static void
track_disp_hdr_cb(const route_head *track)
{
cur_info = &route_info[route_idx];
cur_info->prev_wpt = NULL;
cur_info->total = 0;
current_trk = (route_head *)track;
if (track->rte_waypt_ct <= 0) return;
if (!gtxt_flags.track_header_written) {
gtxt_flags.track_header_written = 1;
gbfprintf(fout, "\r\n\r\nHeader\t%s\r\n", headers[track_header]);
}
print_string("\r\nTrack\t%s\t", current_trk->rte_name ? current_trk->rte_name : "");
print_date_and_time(cur_info->start, 0);
print_date_and_time(cur_info->time, 1);
print_distance(cur_info->length, 0, 1, 0);
print_speed(&cur_info->length, &cur_info->time);
print_string("%s", (track->rte_url != NULL) ? track->rte_url : "");
gbfprintf(fout, "\r\n\r\nHeader\t%s\r\n\r\n", headers[trkpt_header]);
}
static void
track_disp_wpt_cb(const waypoint *wpt)
{
waypoint *prev = cur_info->prev_wpt;
time_t delta;
double dist, depth;
gbfprintf(fout, "Trackpoint\t");
print_position(wpt);
print_date_and_time(wpt->creation_time, 0);
if IS_VALID_ALT(wpt->altitude)
print_distance(wpt->altitude, 1, 0, 0);
gbfprintf(fout, "\t");
depth = WAYPT_GET(wpt, depth, unknown_alt);
if (depth != unknown_alt)
print_distance(depth, 1, 0, 1);
if (prev != NULL) {
float temp;
gbfprintf(fout, "\t");
delta = wpt->creation_time - prev->creation_time;
temp = WAYPT_GET(wpt, temperature, -999);
if (temp != -999)
print_temperature(temp);
gbfprintf(fout, "\t");
dist = waypt_distance_ex(prev, wpt);
print_distance(dist, 0, 1, 0);
print_date_and_time(delta, 1);
print_speed(&dist, &delta);
print_course(prev, wpt);
}
gbfprintf(fout, "\r\n");
cur_info->prev_wpt = (waypoint *)wpt;
}
double do_md_openmm(FILE *fplog,t_commrec *cr,int nfile,const t_filenm fnm[],
const output_env_t oenv, gmx_bool bVerbose,gmx_bool bCompact,
int nstglobalcomm,
gmx_vsite_t *vsite,gmx_constr_t constr,
int stepout,t_inputrec *ir,
gmx_mtop_t *top_global,
t_fcdata *fcd,
t_state *state_global,
t_mdatoms *mdatoms,
t_nrnb *nrnb,gmx_wallcycle_t wcycle,
gmx_edsam_t ed,t_forcerec *fr,
int repl_ex_nst,int repl_ex_seed,
real cpt_period,real max_hours,
const char *deviceOptions,
unsigned long Flags,
gmx_runtime_t *runtime)
{
gmx_mdoutf_t *outf;
gmx_large_int_t step,step_rel;
double run_time;
double t,t0,lam0;
gmx_bool bSimAnn,
bFirstStep,bStateFromTPX,bLastStep,bStartingFromCpt;
gmx_bool bInitStep=TRUE;
gmx_bool do_ene,do_log, do_verbose,
bX,bV,bF,bCPT;
tensor force_vir,shake_vir,total_vir,pres;
int i,m;
int mdof_flags;
rvec mu_tot;
t_vcm *vcm;
int nchkpt=1;
gmx_localtop_t *top;
t_mdebin *mdebin=NULL;
t_state *state=NULL;
rvec *f_global=NULL;
int n_xtc=-1;
rvec *x_xtc=NULL;
gmx_enerdata_t *enerd;
rvec *f=NULL;
gmx_global_stat_t gstat;
gmx_update_t upd=NULL;
t_graph *graph=NULL;
globsig_t gs;
gmx_groups_t *groups;
gmx_ekindata_t *ekind, *ekind_save;
gmx_bool bAppend;
int a0,a1;
matrix lastbox;
real reset_counters=0,reset_counters_now=0;
char sbuf[STEPSTRSIZE],sbuf2[STEPSTRSIZE];
int handled_stop_condition=gmx_stop_cond_none;
const char *ommOptions = NULL;
void *openmmData;
bAppend = (Flags & MD_APPENDFILES);
check_ir_old_tpx_versions(cr,fplog,ir,top_global);
groups = &top_global->groups;
/* Initial values */
init_md(fplog,cr,ir,oenv,&t,&t0,&state_global->lambda,&lam0,
nrnb,top_global,&upd,
nfile,fnm,&outf,&mdebin,
force_vir,shake_vir,mu_tot,&bSimAnn,&vcm,state_global,Flags);
clear_mat(total_vir);
clear_mat(pres);
/* Energy terms and groups */
snew(enerd,1);
init_enerdata(top_global->groups.grps[egcENER].nr,ir->n_flambda,enerd);
snew(f,top_global->natoms);
/* Kinetic energy data */
snew(ekind,1);
init_ekindata(fplog,top_global,&(ir->opts),ekind);
/* needed for iteration of constraints */
snew(ekind_save,1);
init_ekindata(fplog,top_global,&(ir->opts),ekind_save);
/* Copy the cos acceleration to the groups struct */
ekind->cosacc.cos_accel = ir->cos_accel;
gstat = global_stat_init(ir);
debug_gmx();
{
double io = compute_io(ir,top_global->natoms,groups,mdebin->ebin->nener,1);
if ((io > 2000) && MASTER(cr))
fprintf(stderr,
"\nWARNING: This run will generate roughly %.0f Mb of data\n\n",
io);
}
top = gmx_mtop_generate_local_top(top_global,ir);
a0 = 0;
a1 = top_global->natoms;
state = partdec_init_local_state(cr,state_global);
f_global = f;
atoms2md(top_global,ir,0,NULL,a0,a1-a0,mdatoms);
if (vsite)
{
set_vsite_top(vsite,top,mdatoms,cr);
}
if (ir->ePBC != epbcNONE && !ir->bPeriodicMols)
{
graph = mk_graph(fplog,&(top->idef),0,top_global->natoms,FALSE,FALSE);
}
update_mdatoms(mdatoms,state->lambda);
if (deviceOptions[0]=='\0')
{
/* empty options, which should default to OpenMM in this build */
ommOptions=deviceOptions;
}
else
{
if (gmx_strncasecmp(deviceOptions,"OpenMM",6)!=0)
{
gmx_fatal(FARGS, "This Gromacs version currently only works with OpenMM. Use -device \"OpenMM:<options>\"");
}
else
{
ommOptions=strchr(deviceOptions,':');
if (NULL!=ommOptions)
{
/* Increase the pointer to skip the colon */
ommOptions++;
}
}
}
openmmData = openmm_init(fplog, ommOptions, ir, top_global, top, mdatoms, fr, state);
please_cite(fplog,"Friedrichs2009");
if (MASTER(cr))
{
/* Update mdebin with energy history if appending to output files */
if ( Flags & MD_APPENDFILES )
{
restore_energyhistory_from_state(mdebin,&state_global->enerhist);
}
/* Set the initial energy history in state to zero by updating once */
update_energyhistory(&state_global->enerhist,mdebin);
}
if (constr)
{
set_constraints(constr,top,ir,mdatoms,cr);
}
if (!ir->bContinuation)
{
if (mdatoms->cFREEZE && (state->flags & (1<<estV)))
{
/* Set the velocities of frozen particles to zero */
for (i=mdatoms->start; i<mdatoms->start+mdatoms->homenr; i++)
{
for (m=0; m<DIM; m++)
{
if (ir->opts.nFreeze[mdatoms->cFREEZE[i]][m])
{
state->v[i][m] = 0;
}
}
}
}
if (constr)
{
/* Constrain the initial coordinates and velocities */
do_constrain_first(fplog,constr,ir,mdatoms,state,f,
graph,cr,nrnb,fr,top,shake_vir);
}
if (vsite)
{
/* Construct the virtual sites for the initial configuration */
construct_vsites(fplog,vsite,state->x,nrnb,ir->delta_t,NULL,
top->idef.iparams,top->idef.il,
fr->ePBC,fr->bMolPBC,graph,cr,state->box);
}
}
debug_gmx();
if (MASTER(cr))
{
char tbuf[20];
fprintf(fplog,"Initial temperature: %g K\n",enerd->term[F_TEMP]);
fprintf(stderr,"starting mdrun '%s'\n",
*(top_global->name));
if (ir->nsteps >= 0)
{
sprintf(tbuf,"%8.1f",(ir->init_step+ir->nsteps)*ir->delta_t);
}
else
{
sprintf(tbuf,"%s","infinite");
}
if (ir->init_step > 0)
{
fprintf(stderr,"%s steps, %s ps (continuing from step %s, %8.1f ps).\n",
gmx_step_str(ir->init_step+ir->nsteps,sbuf),tbuf,
gmx_step_str(ir->init_step,sbuf2),
ir->init_step*ir->delta_t);
}
else
{
fprintf(stderr,"%s steps, %s ps.\n",
gmx_step_str(ir->nsteps,sbuf),tbuf);
}
}
fprintf(fplog,"\n");
/* Set and write start time */
runtime_start(runtime);
print_date_and_time(fplog,cr->nodeid,"Started mdrun",runtime);
wallcycle_start(wcycle,ewcRUN);
if (fplog)
fprintf(fplog,"\n");
/* safest point to do file checkpointing is here. More general point would be immediately before integrator call */
debug_gmx();
/***********************************************************
*
* Loop over MD steps
*
************************************************************/
/* loop over MD steps or if rerunMD to end of input trajectory */
bFirstStep = TRUE;
/* Skip the first Nose-Hoover integration when we get the state from tpx */
bStateFromTPX = !opt2bSet("-cpi",nfile,fnm);
bInitStep = bFirstStep && bStateFromTPX;
bStartingFromCpt = (Flags & MD_STARTFROMCPT) && bInitStep;
bLastStep = FALSE;
init_global_signals(&gs,cr,ir,repl_ex_nst);
step = ir->init_step;
step_rel = 0;
while (!bLastStep)
{
wallcycle_start(wcycle,ewcSTEP);
GMX_MPE_LOG(ev_timestep1);
bLastStep = (step_rel == ir->nsteps);
t = t0 + step*ir->delta_t;
if (gs.set[eglsSTOPCOND] != 0)
{
bLastStep = TRUE;
}
do_log = do_per_step(step,ir->nstlog) || bFirstStep || bLastStep;
do_verbose = bVerbose &&
(step % stepout == 0 || bFirstStep || bLastStep);
if (MASTER(cr) && do_log)
{
print_ebin_header(fplog,step,t,state->lambda);
}
clear_mat(force_vir);
GMX_MPE_LOG(ev_timestep2);
/* We write a checkpoint at this MD step when:
* either when we signalled through gs (in OpenMM NS works different),
* or at the last step (but not when we do not want confout),
* but never at the first step.
*/
bCPT = ((gs.set[eglsCHKPT] ||
(bLastStep && (Flags & MD_CONFOUT))) &&
step > ir->init_step );
if (bCPT)
{
gs.set[eglsCHKPT] = 0;
}
/* Now we have the energies and forces corresponding to the
* coordinates at time t. We must output all of this before
* the update.
* for RerunMD t is read from input trajectory
*/
GMX_MPE_LOG(ev_output_start);
mdof_flags = 0;
if (do_per_step(step,ir->nstxout))
{
mdof_flags |= MDOF_X;
}
if (do_per_step(step,ir->nstvout))
{
mdof_flags |= MDOF_V;
}
if (do_per_step(step,ir->nstfout))
{
mdof_flags |= MDOF_F;
}
if (do_per_step(step,ir->nstxtcout))
{
mdof_flags |= MDOF_XTC;
}
if (bCPT)
{
mdof_flags |= MDOF_CPT;
};
do_ene = (do_per_step(step,ir->nstenergy) || bLastStep);
if (mdof_flags != 0 || do_ene || do_log)
{
wallcycle_start(wcycle,ewcTRAJ);
bF = (mdof_flags & MDOF_F);
bX = (mdof_flags & (MDOF_X | MDOF_XTC | MDOF_CPT));
bV = (mdof_flags & (MDOF_V | MDOF_CPT));
openmm_copy_state(openmmData, state, &t, f, enerd, bX, bV, bF, do_ene);
upd_mdebin(mdebin, FALSE,TRUE,
t,mdatoms->tmass,enerd,state,lastbox,
shake_vir,force_vir,total_vir,pres,
ekind,mu_tot,constr);
print_ebin(outf->fp_ene,do_ene,FALSE,FALSE,do_log?fplog:NULL,
step,t,
eprNORMAL,bCompact,mdebin,fcd,groups,&(ir->opts));
write_traj(fplog,cr,outf,mdof_flags,top_global,
step,t,state,state_global,f,f_global,&n_xtc,&x_xtc);
if (bCPT)
{
nchkpt++;
bCPT = FALSE;
}
debug_gmx();
if (bLastStep && step_rel == ir->nsteps &&
(Flags & MD_CONFOUT) && MASTER(cr))
{
/* x and v have been collected in write_traj,
* because a checkpoint file will always be written
* at the last step.
*/
fprintf(stderr,"\nWriting final coordinates.\n");
if (ir->ePBC != epbcNONE && !ir->bPeriodicMols)
{
/* Make molecules whole only for confout writing */
do_pbc_mtop(fplog,ir->ePBC,state->box,top_global,state_global->x);
}
write_sto_conf_mtop(ftp2fn(efSTO,nfile,fnm),
*top_global->name,top_global,
state_global->x,state_global->v,
ir->ePBC,state->box);
debug_gmx();
}
wallcycle_stop(wcycle,ewcTRAJ);
}
GMX_MPE_LOG(ev_output_finish);
/* Determine the wallclock run time up till now */
run_time = gmx_gettime() - (double)runtime->real;
/* Check whether everything is still allright */
if (((int)gmx_get_stop_condition() > handled_stop_condition)
#ifdef GMX_THREADS
&& MASTER(cr)
#endif
)
{
/* this is just make gs.sig compatible with the hack
of sending signals around by MPI_Reduce with together with
other floats */
/* NOTE: this only works for serial code. For code that allows
MPI nodes to propagate their condition, see kernel/md.c*/
if ( gmx_get_stop_condition() == gmx_stop_cond_next_ns )
gs.set[eglsSTOPCOND]=1;
if ( gmx_get_stop_condition() == gmx_stop_cond_next )
gs.set[eglsSTOPCOND]=1;
/* < 0 means stop at next step, > 0 means stop at next NS step */
if (fplog)
{
fprintf(fplog,
"\n\nReceived the %s signal, stopping at the next %sstep\n\n",
gmx_get_signal_name(),
gs.sig[eglsSTOPCOND]==1 ? "NS " : "");
fflush(fplog);
}
fprintf(stderr,
"\n\nReceived the %s signal, stopping at the next %sstep\n\n",
gmx_get_signal_name(),
gs.sig[eglsSTOPCOND]==1 ? "NS " : "");
fflush(stderr);
handled_stop_condition=(int)gmx_get_stop_condition();
}
else if (MASTER(cr) &&
(max_hours > 0 && run_time > max_hours*60.0*60.0*0.99) &&
gs.set[eglsSTOPCOND] == 0)
{
/* Signal to terminate the run */
gs.set[eglsSTOPCOND] = 1;
if (fplog)
{
fprintf(fplog,"\nStep %s: Run time exceeded %.3f hours, will terminate the run\n",gmx_step_str(step,sbuf),max_hours*0.99);
}
fprintf(stderr, "\nStep %s: Run time exceeded %.3f hours, will terminate the run\n",gmx_step_str(step,sbuf),max_hours*0.99);
}
/* checkpoints */
if (MASTER(cr) && (cpt_period >= 0 &&
(cpt_period == 0 ||
run_time >= nchkpt*cpt_period*60.0)) &&
gs.set[eglsCHKPT] == 0)
{
gs.set[eglsCHKPT] = 1;
}
/* Time for performance */
if (((step % stepout) == 0) || bLastStep)
{
runtime_upd_proc(runtime);
}
if (do_per_step(step,ir->nstlog))
{
if (fflush(fplog) != 0)
{
gmx_fatal(FARGS,"Cannot flush logfile - maybe you are out of quota?");
}
}
/* Remaining runtime */
if (MULTIMASTER(cr) && (do_verbose || gmx_got_usr_signal() ))
{
print_time(stderr,runtime,step,ir,cr);
}
bFirstStep = FALSE;
bInitStep = FALSE;
bStartingFromCpt = FALSE;
step++;
step_rel++;
openmm_take_one_step(openmmData);
}
/* End of main MD loop */
debug_gmx();
/* Stop the time */
runtime_end(runtime);
if (MASTER(cr))
{
if (ir->nstcalcenergy > 0)
{
print_ebin(outf->fp_ene,FALSE,FALSE,FALSE,fplog,step,t,
eprAVER,FALSE,mdebin,fcd,groups,&(ir->opts));
}
}
openmm_cleanup(fplog, openmmData);
done_mdoutf(outf);
debug_gmx();
runtime->nsteps_done = step_rel;
return 0;
}
static void
write_waypt(const waypoint *wpt)
{
unsigned char wpt_class;
garmin_fs_p gmsd;
char *wpt_type;
char *dspl_mode;
const char *country;
double x;
int i, icon, dynamic;
char *icon_descr;
gmsd = GMSD_FIND(wpt);
i = GMSD_GET(display, 0);
if (i > GT_DISPLAY_MODE_MAX) i = 0;
dspl_mode = gt_display_mode_names[i];
wpt_class = GMSD_GET(wpt_class, 0);
if (wpt_class <= gt_waypt_class_map_line)
wpt_type = gt_waypt_class_names[wpt_class];
else
wpt_type = gt_waypt_class_names[0];
gbfprintf(fout, "Waypoint\t%s\t", (wpt->shortname) ? wpt->shortname : "");
if (wpt_class <= gt_waypt_class_airport_ndb) {
char *temp = wpt->notes;
if (temp == NULL) {
if (wpt->description && (strcmp(wpt->description, wpt->shortname) != 0))
temp = wpt->description;
else
temp = "";
}
print_string("%s\t", temp);
}
else
gbfprintf(fout, "\t");
gbfprintf(fout, "%s\t", wpt_type);
print_position(wpt);
if IS_VALID_ALT(wpt->altitude)
print_distance(wpt->altitude, 1, 0, 0);
gbfprintf(fout, "\t");
x = WAYPT_GET(wpt, depth, unknown_alt);
if (x != unknown_alt)
print_distance(x, 1, 0, 1);
gbfprintf(fout, "\t");
x = WAYPT_GET(wpt, proximity, unknown_alt);
if (x != unknown_alt)
print_distance(x, 0, 0, 0);
gbfprintf(fout, "\t");
x = WAYPT_GET(wpt, temperature, -999);
if (x != -999)
print_temperature(x);
gbfprintf(fout, "\t%s\t", dspl_mode);
gbfprintf(fout, "Unknown\t"); /* Color is fixed: Unknown */
icon = GMSD_GET(icon, -1);
if (icon == -1) {
icon = gt_find_icon_number_from_desc(wpt->icon_descr, GDB);
}
icon_descr = gt_find_desc_from_icon_number(icon, GDB, &dynamic);
print_string("%s\t", icon_descr);
if (dynamic) xfree(icon_descr);
print_string("%s\t", GMSD_GET(facility, ""));
print_string("%s\t", GMSD_GET(city, ""));
print_string("%s\t", GMSD_GET(state, ""));
country = gt_get_icao_country(GMSD_GET(cc, ""));
print_string("%s\t", (country != NULL) ? country : "");
print_date_and_time(wpt->creation_time, 0);
print_string("%s\t", wpt->url ? wpt->url : "");
print_categories(GMSD_GET(category, 0));
gbfprintf(fout, "\r\n");
}
double do_tpi(FILE *fplog,t_commrec *cr,
int nfile, const t_filenm fnm[],
const output_env_t oenv, gmx_bool bVerbose,gmx_bool bCompact,
int nstglobalcomm,
gmx_vsite_t *vsite,gmx_constr_t constr,
int stepout,
t_inputrec *inputrec,
gmx_mtop_t *top_global,t_fcdata *fcd,
t_state *state,
t_mdatoms *mdatoms,
t_nrnb *nrnb,gmx_wallcycle_t wcycle,
gmx_edsam_t ed,
t_forcerec *fr,
int repl_ex_nst,int repl_ex_seed,
real cpt_period,real max_hours,
const char *deviceOptions,
unsigned long Flags,
gmx_runtime_t *runtime)
{
const char *TPI="Test Particle Insertion";
gmx_localtop_t *top;
gmx_groups_t *groups;
gmx_enerdata_t *enerd;
rvec *f;
real lambda,t,temp,beta,drmax,epot;
double embU,sum_embU,*sum_UgembU,V,V_all,VembU_all;
t_trxstatus *status;
t_trxframe rerun_fr;
gmx_bool bDispCorr,bCharge,bRFExcl,bNotLastFrame,bStateChanged,bNS,bOurStep;
tensor force_vir,shake_vir,vir,pres;
int cg_tp,a_tp0,a_tp1,ngid,gid_tp,nener,e;
rvec *x_mol;
rvec mu_tot,x_init,dx,x_tp;
int nnodes,frame,nsteps,step;
int i,start,end;
gmx_rng_t tpi_rand;
FILE *fp_tpi=NULL;
char *ptr,*dump_pdb,**leg,str[STRLEN],str2[STRLEN];
double dbl,dump_ener;
gmx_bool bCavity;
int nat_cavity=0,d;
real *mass_cavity=NULL,mass_tot;
int nbin;
double invbinw,*bin,refvolshift,logV,bUlogV;
real dvdl,prescorr,enercorr,dvdlcorr;
gmx_bool bEnergyOutOfBounds;
const char *tpid_leg[2]={"direct","reweighted"};
/* Since there is no upper limit to the insertion energies,
* we need to set an upper limit for the distribution output.
*/
real bU_bin_limit = 50;
real bU_logV_bin_limit = bU_bin_limit + 10;
nnodes = cr->nnodes;
top = gmx_mtop_generate_local_top(top_global,inputrec);
groups = &top_global->groups;
bCavity = (inputrec->eI == eiTPIC);
if (bCavity) {
ptr = getenv("GMX_TPIC_MASSES");
if (ptr == NULL) {
nat_cavity = 1;
} else {
/* Read (multiple) masses from env var GMX_TPIC_MASSES,
* The center of mass of the last atoms is then used for TPIC.
*/
nat_cavity = 0;
while (sscanf(ptr,"%lf%n",&dbl,&i) > 0) {
srenew(mass_cavity,nat_cavity+1);
mass_cavity[nat_cavity] = dbl;
fprintf(fplog,"mass[%d] = %f\n",
nat_cavity+1,mass_cavity[nat_cavity]);
nat_cavity++;
ptr += i;
}
if (nat_cavity == 0)
gmx_fatal(FARGS,"Found %d masses in GMX_TPIC_MASSES",nat_cavity);
}
}
/*
init_em(fplog,TPI,inputrec,&lambda,nrnb,mu_tot,
state->box,fr,mdatoms,top,cr,nfile,fnm,NULL,NULL);*/
/* We never need full pbc for TPI */
fr->ePBC = epbcXYZ;
/* Determine the temperature for the Boltzmann weighting */
temp = inputrec->opts.ref_t[0];
if (fplog) {
for(i=1; (i<inputrec->opts.ngtc); i++) {
if (inputrec->opts.ref_t[i] != temp) {
fprintf(fplog,"\nWARNING: The temperatures of the different temperature coupling groups are not identical\n\n");
fprintf(stderr,"\nWARNING: The temperatures of the different temperature coupling groups are not identical\n\n");
}
}
fprintf(fplog,
"\n The temperature for test particle insertion is %.3f K\n\n",
temp);
}
beta = 1.0/(BOLTZ*temp);
/* Number of insertions per frame */
nsteps = inputrec->nsteps;
/* Use the same neighborlist with more insertions points
* in a sphere of radius drmax around the initial point
*/
/* This should be a proper mdp parameter */
drmax = inputrec->rtpi;
/* An environment variable can be set to dump all configurations
* to pdb with an insertion energy <= this value.
*/
dump_pdb = getenv("GMX_TPI_DUMP");
dump_ener = 0;
if (dump_pdb)
sscanf(dump_pdb,"%lf",&dump_ener);
atoms2md(top_global,inputrec,0,NULL,0,top_global->natoms,mdatoms);
update_mdatoms(mdatoms,inputrec->init_lambda);
snew(enerd,1);
init_enerdata(groups->grps[egcENER].nr,inputrec->n_flambda,enerd);
snew(f,top_global->natoms);
/* Print to log file */
runtime_start(runtime);
print_date_and_time(fplog,cr->nodeid,
"Started Test Particle Insertion",runtime);
wallcycle_start(wcycle,ewcRUN);
/* The last charge group is the group to be inserted */
cg_tp = top->cgs.nr - 1;
a_tp0 = top->cgs.index[cg_tp];
a_tp1 = top->cgs.index[cg_tp+1];
if (debug)
fprintf(debug,"TPI cg %d, atoms %d-%d\n",cg_tp,a_tp0,a_tp1);
if (a_tp1 - a_tp0 > 1 &&
(inputrec->rlist < inputrec->rcoulomb ||
inputrec->rlist < inputrec->rvdw))
gmx_fatal(FARGS,"Can not do TPI for multi-atom molecule with a twin-range cut-off");
snew(x_mol,a_tp1-a_tp0);
bDispCorr = (inputrec->eDispCorr != edispcNO);
bCharge = FALSE;
for(i=a_tp0; i<a_tp1; i++) {
/* Copy the coordinates of the molecule to be insterted */
copy_rvec(state->x[i],x_mol[i-a_tp0]);
/* Check if we need to print electrostatic energies */
bCharge |= (mdatoms->chargeA[i] != 0 ||
(mdatoms->chargeB && mdatoms->chargeB[i] != 0));
}
bRFExcl = (bCharge && EEL_RF(fr->eeltype) && fr->eeltype!=eelRF_NEC);
calc_cgcm(fplog,cg_tp,cg_tp+1,&(top->cgs),state->x,fr->cg_cm);
if (bCavity) {
if (norm(fr->cg_cm[cg_tp]) > 0.5*inputrec->rlist && fplog) {
fprintf(fplog, "WARNING: Your TPI molecule is not centered at 0,0,0\n");
fprintf(stderr,"WARNING: Your TPI molecule is not centered at 0,0,0\n");
}
} else {
/* Center the molecule to be inserted at zero */
for(i=0; i<a_tp1-a_tp0; i++)
rvec_dec(x_mol[i],fr->cg_cm[cg_tp]);
}
if (fplog) {
fprintf(fplog,"\nWill insert %d atoms %s partial charges\n",
a_tp1-a_tp0,bCharge ? "with" : "without");
fprintf(fplog,"\nWill insert %d times in each frame of %s\n",
nsteps,opt2fn("-rerun",nfile,fnm));
}
if (!bCavity)
{
if (inputrec->nstlist > 1)
{
if (drmax==0 && a_tp1-a_tp0==1)
{
gmx_fatal(FARGS,"Re-using the neighborlist %d times for insertions of a single atom in a sphere of radius %f does not make sense",inputrec->nstlist,drmax);
}
if (fplog)
{
fprintf(fplog,"Will use the same neighborlist for %d insertions in a sphere of radius %f\n",inputrec->nstlist,drmax);
}
}
}
else
{
if (fplog)
{
fprintf(fplog,"Will insert randomly in a sphere of radius %f around the center of the cavity\n",drmax);
}
}
ngid = groups->grps[egcENER].nr;
gid_tp = GET_CGINFO_GID(fr->cginfo[cg_tp]);
nener = 1 + ngid;
if (bDispCorr)
nener += 1;
if (bCharge) {
nener += ngid;
if (bRFExcl)
nener += 1;
if (EEL_FULL(fr->eeltype))
nener += 1;
}
snew(sum_UgembU,nener);
/* Initialize random generator */
tpi_rand = gmx_rng_init(inputrec->ld_seed);
if (MASTER(cr)) {
fp_tpi = xvgropen(opt2fn("-tpi",nfile,fnm),
"TPI energies","Time (ps)",
"(kJ mol\\S-1\\N) / (nm\\S3\\N)",oenv);
xvgr_subtitle(fp_tpi,"f. are averages over one frame",oenv);
snew(leg,4+nener);
e = 0;
sprintf(str,"-kT log(<Ve\\S-\\betaU\\N>/<V>)");
leg[e++] = strdup(str);
sprintf(str,"f. -kT log<e\\S-\\betaU\\N>");
leg[e++] = strdup(str);
sprintf(str,"f. <e\\S-\\betaU\\N>");
leg[e++] = strdup(str);
sprintf(str,"f. V");
leg[e++] = strdup(str);
sprintf(str,"f. <Ue\\S-\\betaU\\N>");
leg[e++] = strdup(str);
for(i=0; i<ngid; i++) {
sprintf(str,"f. <U\\sVdW %s\\Ne\\S-\\betaU\\N>",
*(groups->grpname[groups->grps[egcENER].nm_ind[i]]));
leg[e++] = strdup(str);
}
if (bDispCorr) {
sprintf(str,"f. <U\\sdisp c\\Ne\\S-\\betaU\\N>");
leg[e++] = strdup(str);
}
if (bCharge) {
for(i=0; i<ngid; i++) {
sprintf(str,"f. <U\\sCoul %s\\Ne\\S-\\betaU\\N>",
*(groups->grpname[groups->grps[egcENER].nm_ind[i]]));
leg[e++] = strdup(str);
}
if (bRFExcl) {
sprintf(str,"f. <U\\sRF excl\\Ne\\S-\\betaU\\N>");
leg[e++] = strdup(str);
}
if (EEL_FULL(fr->eeltype)) {
sprintf(str,"f. <U\\sCoul recip\\Ne\\S-\\betaU\\N>");
leg[e++] = strdup(str);
}
}
xvgr_legend(fp_tpi,4+nener,(const char**)leg,oenv);
for(i=0; i<4+nener; i++)
sfree(leg[i]);
sfree(leg);
}
clear_rvec(x_init);
V_all = 0;
VembU_all = 0;
invbinw = 10;
nbin = 10;
snew(bin,nbin);
bNotLastFrame = read_first_frame(oenv,&status,opt2fn("-rerun",nfile,fnm),
&rerun_fr,TRX_NEED_X);
frame = 0;
if (rerun_fr.natoms - (bCavity ? nat_cavity : 0) !=
mdatoms->nr - (a_tp1 - a_tp0))
gmx_fatal(FARGS,"Number of atoms in trajectory (%d)%s "
"is not equal the number in the run input file (%d) "
"minus the number of atoms to insert (%d)\n",
rerun_fr.natoms,bCavity ? " minus one" : "",
mdatoms->nr,a_tp1-a_tp0);
refvolshift = log(det(rerun_fr.box));
#if ( defined(GMX_IA32_SSE) || defined(GMX_X86_64_SSE) || defined(GMX_X86_64_SSE2) )
/* Make sure we don't detect SSE overflow generated before this point */
gmx_mm_check_and_reset_overflow();
#endif
while (bNotLastFrame)
{
lambda = rerun_fr.lambda;
t = rerun_fr.time;
sum_embU = 0;
for(e=0; e<nener; e++)
{
sum_UgembU[e] = 0;
}
/* Copy the coordinates from the input trajectory */
for(i=0; i<rerun_fr.natoms; i++)
{
copy_rvec(rerun_fr.x[i],state->x[i]);
}
V = det(rerun_fr.box);
logV = log(V);
bStateChanged = TRUE;
bNS = TRUE;
for(step=0; step<nsteps; step++)
{
/* In parallel all nodes generate all random configurations.
* In that way the result is identical to a single cpu tpi run.
*/
if (!bCavity)
{
/* Random insertion in the whole volume */
bNS = (step % inputrec->nstlist == 0);
if (bNS)
{
/* Generate a random position in the box */
x_init[XX] = gmx_rng_uniform_real(tpi_rand)*state->box[XX][XX];
x_init[YY] = gmx_rng_uniform_real(tpi_rand)*state->box[YY][YY];
x_init[ZZ] = gmx_rng_uniform_real(tpi_rand)*state->box[ZZ][ZZ];
}
if (inputrec->nstlist == 1)
{
copy_rvec(x_init,x_tp);
}
else
{
/* Generate coordinates within |dx|=drmax of x_init */
do
{
dx[XX] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax;
dx[YY] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax;
dx[ZZ] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax;
}
while (norm2(dx) > drmax*drmax);
rvec_add(x_init,dx,x_tp);
}
}
else
{
/* Random insertion around a cavity location
* given by the last coordinate of the trajectory.
*/
if (step == 0)
{
if (nat_cavity == 1)
{
/* Copy the location of the cavity */
copy_rvec(rerun_fr.x[rerun_fr.natoms-1],x_init);
}
else
{
/* Determine the center of mass of the last molecule */
clear_rvec(x_init);
mass_tot = 0;
for(i=0; i<nat_cavity; i++)
{
for(d=0; d<DIM; d++)
{
x_init[d] +=
mass_cavity[i]*rerun_fr.x[rerun_fr.natoms-nat_cavity+i][d];
}
mass_tot += mass_cavity[i];
}
for(d=0; d<DIM; d++)
{
x_init[d] /= mass_tot;
}
}
}
/* Generate coordinates within |dx|=drmax of x_init */
do
{
dx[XX] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax;
dx[YY] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax;
dx[ZZ] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax;
}
while (norm2(dx) > drmax*drmax);
rvec_add(x_init,dx,x_tp);
}
if (a_tp1 - a_tp0 == 1)
{
/* Insert a single atom, just copy the insertion location */
copy_rvec(x_tp,state->x[a_tp0]);
}
else
{
/* Copy the coordinates from the top file */
for(i=a_tp0; i<a_tp1; i++)
{
copy_rvec(x_mol[i-a_tp0],state->x[i]);
}
/* Rotate the molecule randomly */
rotate_conf(a_tp1-a_tp0,state->x+a_tp0,NULL,
2*M_PI*gmx_rng_uniform_real(tpi_rand),
2*M_PI*gmx_rng_uniform_real(tpi_rand),
2*M_PI*gmx_rng_uniform_real(tpi_rand));
/* Shift to the insertion location */
for(i=a_tp0; i<a_tp1; i++)
{
rvec_inc(state->x[i],x_tp);
}
}
/* Check if this insertion belongs to this node */
bOurStep = TRUE;
if (PAR(cr))
{
switch (inputrec->eI)
{
case eiTPI:
bOurStep = ((step / inputrec->nstlist) % nnodes == cr->nodeid);
break;
case eiTPIC:
bOurStep = (step % nnodes == cr->nodeid);
break;
default:
gmx_fatal(FARGS,"Unknown integrator %s",ei_names[inputrec->eI]);
}
}
if (bOurStep)
{
/* Clear some matrix variables */
clear_mat(force_vir);
clear_mat(shake_vir);
clear_mat(vir);
clear_mat(pres);
/* Set the charge group center of mass of the test particle */
copy_rvec(x_init,fr->cg_cm[top->cgs.nr-1]);
/* Calc energy (no forces) on new positions.
* Since we only need the intermolecular energy
* and the RF exclusion terms of the inserted molecule occur
* within a single charge group we can pass NULL for the graph.
* This also avoids shifts that would move charge groups
* out of the box.
*
* Some checks above ensure than we can not have
* twin-range interactions together with nstlist > 1,
* therefore we do not need to remember the LR energies.
*/
/* Make do_force do a single node force calculation */
cr->nnodes = 1;
do_force(fplog,cr,inputrec,
step,nrnb,wcycle,top,top_global,&top_global->groups,
rerun_fr.box,state->x,&state->hist,
f,force_vir,mdatoms,enerd,fcd,
lambda,NULL,fr,NULL,mu_tot,t,NULL,NULL,FALSE,
GMX_FORCE_NONBONDED |
(bNS ? GMX_FORCE_NS | GMX_FORCE_DOLR : 0) |
(bStateChanged ? GMX_FORCE_STATECHANGED : 0));
cr->nnodes = nnodes;
bStateChanged = FALSE;
bNS = FALSE;
/* Calculate long range corrections to pressure and energy */
calc_dispcorr(fplog,inputrec,fr,step,top_global->natoms,rerun_fr.box,
lambda,pres,vir,&prescorr,&enercorr,&dvdlcorr);
/* figure out how to rearrange the next 4 lines MRS 8/4/2009 */
enerd->term[F_DISPCORR] = enercorr;
enerd->term[F_EPOT] += enercorr;
enerd->term[F_PRES] += prescorr;
enerd->term[F_DVDL] += dvdlcorr;
epot = enerd->term[F_EPOT];
bEnergyOutOfBounds = FALSE;
#if ( defined(GMX_IA32_SSE) || defined(GMX_X86_64_SSE) || defined(GMX_X86_64_SSE2) )
/* With SSE the energy can overflow, check for this */
if (gmx_mm_check_and_reset_overflow())
{
if (debug)
{
fprintf(debug,"Found an SSE overflow, assuming the energy is out of bounds\n");
}
bEnergyOutOfBounds = TRUE;
}
#endif
/* If the compiler doesn't optimize this check away
* we catch the NAN energies.
* The epot>GMX_REAL_MAX check catches inf values,
* which should nicely result in embU=0 through the exp below,
* but it does not hurt to check anyhow.
*/
/* Non-bonded Interaction usually diverge at r=0.
* With tabulated interaction functions the first few entries
* should be capped in a consistent fashion between
* repulsion, dispersion and Coulomb to avoid accidental
* negative values in the total energy.
* The table generation code in tables.c does this.
* With user tbales the user should take care of this.
*/
if (epot != epot || epot > GMX_REAL_MAX)
{
bEnergyOutOfBounds = TRUE;
}
if (bEnergyOutOfBounds)
{
if (debug)
{
fprintf(debug,"\n time %.3f, step %d: non-finite energy %f, using exp(-bU)=0\n",t,step,epot);
}
embU = 0;
}
else
{
embU = exp(-beta*epot);
sum_embU += embU;
/* Determine the weighted energy contributions of each energy group */
e = 0;
sum_UgembU[e++] += epot*embU;
if (fr->bBHAM)
{
for(i=0; i<ngid; i++)
{
sum_UgembU[e++] +=
(enerd->grpp.ener[egBHAMSR][GID(i,gid_tp,ngid)] +
enerd->grpp.ener[egBHAMLR][GID(i,gid_tp,ngid)])*embU;
}
}
else
{
for(i=0; i<ngid; i++)
{
sum_UgembU[e++] +=
(enerd->grpp.ener[egLJSR][GID(i,gid_tp,ngid)] +
enerd->grpp.ener[egLJLR][GID(i,gid_tp,ngid)])*embU;
}
}
if (bDispCorr)
{
sum_UgembU[e++] += enerd->term[F_DISPCORR]*embU;
}
if (bCharge)
{
for(i=0; i<ngid; i++)
{
sum_UgembU[e++] +=
(enerd->grpp.ener[egCOULSR][GID(i,gid_tp,ngid)] +
enerd->grpp.ener[egCOULLR][GID(i,gid_tp,ngid)])*embU;
}
if (bRFExcl)
{
sum_UgembU[e++] += enerd->term[F_RF_EXCL]*embU;
}
if (EEL_FULL(fr->eeltype))
{
sum_UgembU[e++] += enerd->term[F_COUL_RECIP]*embU;
}
}
}
if (embU == 0 || beta*epot > bU_bin_limit)
{
bin[0]++;
}
else
{
i = (int)((bU_logV_bin_limit
- (beta*epot - logV + refvolshift))*invbinw
+ 0.5);
if (i < 0)
{
i = 0;
}
if (i >= nbin)
{
realloc_bins(&bin,&nbin,i+10);
}
bin[i]++;
}
if(fr->adress_do_drift ||fr->adress_icor == eAdressICThermoForce ){
sum_UgembU[e++] += enerd->term[F_ADR_DELTU]*embU;
}
if (debug)
{
fprintf(debug,"TPI %7d %12.5e %12.5f %12.5f %12.5f\n",
step,epot,x_tp[XX],x_tp[YY],x_tp[ZZ]);
}
if (dump_pdb && epot <= dump_ener)
{
sprintf(str,"t%g_step%d.pdb",t,step);
sprintf(str2,"t: %f step %d ener: %f",t,step,epot);
write_sto_conf_mtop(str,str2,top_global,state->x,state->v,
inputrec->ePBC,state->box);
}
}
}
if (PAR(cr))
{
/* When running in parallel sum the energies over the processes */
gmx_sumd(1, &sum_embU, cr);
gmx_sumd(nener,sum_UgembU,cr);
}
frame++;
V_all += V;
VembU_all += V*sum_embU/nsteps;
if (fp_tpi)
{
if (bVerbose || frame%10==0 || frame<10)
{
fprintf(stderr,"mu %10.3e <mu> %10.3e\n",
-log(sum_embU/nsteps)/beta,-log(VembU_all/V_all)/beta);
}
fprintf(fp_tpi,"%10.3f %12.5e %12.5e %12.5e %12.5e",
t,
VembU_all==0 ? 20/beta : -log(VembU_all/V_all)/beta,
sum_embU==0 ? 20/beta : -log(sum_embU/nsteps)/beta,
sum_embU/nsteps,V);
for(e=0; e<nener; e++)
{
fprintf(fp_tpi," %12.5e",sum_UgembU[e]/nsteps);
}
fprintf(fp_tpi,"\n");
fflush(fp_tpi);
}
bNotLastFrame = read_next_frame(oenv, status,&rerun_fr);
} /* End of the loop */
runtime_end(runtime);
close_trj(status);
if (fp_tpi != NULL)
{
gmx_fio_fclose(fp_tpi);
}
if (fplog != NULL)
{
fprintf(fplog,"\n");
fprintf(fplog," <V> = %12.5e nm^3\n",V_all/frame);
fprintf(fplog," <mu> = %12.5e kJ/mol\n",-log(VembU_all/V_all)/beta);
}
/* Write the Boltzmann factor histogram */
if (PAR(cr))
{
/* When running in parallel sum the bins over the processes */
i = nbin;
global_max(cr,&i);
realloc_bins(&bin,&nbin,i);
gmx_sumd(nbin,bin,cr);
}
if (MASTER(cr))
{
fp_tpi = xvgropen(opt2fn("-tpid",nfile,fnm),
"TPI energy distribution",
"\\betaU - log(V/<V>)","count",oenv);
sprintf(str,"number \\betaU > %g: %9.3e",bU_bin_limit,bin[0]);
xvgr_subtitle(fp_tpi,str,oenv);
xvgr_legend(fp_tpi,2,(const char **)tpid_leg,oenv);
for(i=nbin-1; i>0; i--)
{
bUlogV = -i/invbinw + bU_logV_bin_limit - refvolshift + log(V_all/frame);
fprintf(fp_tpi,"%6.2f %10d %12.5e\n",
bUlogV,
(int)(bin[i]+0.5),
bin[i]*exp(-bUlogV)*V_all/VembU_all);
}
gmx_fio_fclose(fp_tpi);
}
sfree(bin);
sfree(sum_UgembU);
runtime->nsteps_done = frame*inputrec->nsteps;
return 0;
}
int mdrunner(gmx_hw_opt_t *hw_opt,
FILE *fplog, t_commrec *cr, int nfile,
const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
gmx_bool bCompact, int nstglobalcomm,
ivec ddxyz, int dd_node_order, real rdd, real rconstr,
const char *dddlb_opt, real dlb_scale,
const char *ddcsx, const char *ddcsy, const char *ddcsz,
const char *nbpu_opt, int nstlist_cmdline,
gmx_int64_t nsteps_cmdline, int nstepout, int resetstep,
int gmx_unused nmultisim, int repl_ex_nst, int repl_ex_nex,
int repl_ex_seed, real pforce, real cpt_period, real max_hours,
int imdport, unsigned long Flags)
{
gmx_bool bForceUseGPU, bTryUseGPU, bRerunMD;
t_inputrec *inputrec;
t_state *state = NULL;
matrix box;
gmx_ddbox_t ddbox = {0};
int npme_major, npme_minor;
t_nrnb *nrnb;
gmx_mtop_t *mtop = NULL;
t_mdatoms *mdatoms = NULL;
t_forcerec *fr = NULL;
t_fcdata *fcd = NULL;
real ewaldcoeff_q = 0;
real ewaldcoeff_lj = 0;
struct gmx_pme_t **pmedata = NULL;
gmx_vsite_t *vsite = NULL;
gmx_constr_t constr;
int nChargePerturbed = -1, nTypePerturbed = 0, status;
gmx_wallcycle_t wcycle;
gmx_bool bReadEkin;
gmx_walltime_accounting_t walltime_accounting = NULL;
int rc;
gmx_int64_t reset_counters;
gmx_edsam_t ed = NULL;
int nthreads_pme = 1;
int nthreads_pp = 1;
gmx_membed_t membed = NULL;
gmx_hw_info_t *hwinfo = NULL;
/* The master rank decides early on bUseGPU and broadcasts this later */
gmx_bool bUseGPU = FALSE;
/* CAUTION: threads may be started later on in this function, so
cr doesn't reflect the final parallel state right now */
snew(inputrec, 1);
snew(mtop, 1);
if (Flags & MD_APPENDFILES)
{
fplog = NULL;
}
bRerunMD = (Flags & MD_RERUN);
bForceUseGPU = (strncmp(nbpu_opt, "gpu", 3) == 0);
bTryUseGPU = (strncmp(nbpu_opt, "auto", 4) == 0) || bForceUseGPU;
/* Detect hardware, gather information. This is an operation that is
* global for this process (MPI rank). */
hwinfo = gmx_detect_hardware(fplog, cr, bTryUseGPU);
gmx_print_detected_hardware(fplog, cr, hwinfo);
if (fplog != NULL)
{
/* Print references after all software/hardware printing */
please_cite(fplog, "Abraham2015");
please_cite(fplog, "Pall2015");
please_cite(fplog, "Pronk2013");
please_cite(fplog, "Hess2008b");
please_cite(fplog, "Spoel2005a");
please_cite(fplog, "Lindahl2001a");
please_cite(fplog, "Berendsen95a");
}
snew(state, 1);
if (SIMMASTER(cr))
{
/* Read (nearly) all data required for the simulation */
read_tpx_state(ftp2fn(efTPR, nfile, fnm), inputrec, state, NULL, mtop);
if (inputrec->cutoff_scheme == ecutsVERLET)
{
/* Here the master rank decides if all ranks will use GPUs */
bUseGPU = (hwinfo->gpu_info.n_dev_compatible > 0 ||
getenv("GMX_EMULATE_GPU") != NULL);
/* TODO add GPU kernels for this and replace this check by:
* (bUseGPU && (ir->vdwtype == evdwPME &&
* ir->ljpme_combination_rule == eljpmeLB))
* update the message text and the content of nbnxn_acceleration_supported.
*/
if (bUseGPU &&
!nbnxn_gpu_acceleration_supported(fplog, cr, inputrec, bRerunMD))
{
/* Fallback message printed by nbnxn_acceleration_supported */
if (bForceUseGPU)
{
gmx_fatal(FARGS, "GPU acceleration requested, but not supported with the given input settings");
}
bUseGPU = FALSE;
}
prepare_verlet_scheme(fplog, cr,
inputrec, nstlist_cmdline, mtop, state->box,
bUseGPU);
}
else
{
if (nstlist_cmdline > 0)
{
gmx_fatal(FARGS, "Can not set nstlist with the group cut-off scheme");
}
if (hwinfo->gpu_info.n_dev_compatible > 0)
{
md_print_warn(cr, fplog,
"NOTE: GPU(s) found, but the current simulation can not use GPUs\n"
" To use a GPU, set the mdp option: cutoff-scheme = Verlet\n");
}
if (bForceUseGPU)
{
gmx_fatal(FARGS, "GPU requested, but can't be used without cutoff-scheme=Verlet");
}
#ifdef GMX_TARGET_BGQ
md_print_warn(cr, fplog,
"NOTE: There is no SIMD implementation of the group scheme kernels on\n"
" BlueGene/Q. You will observe better performance from using the\n"
" Verlet cut-off scheme.\n");
#endif
}
if (inputrec->eI == eiSD2)
{
md_print_warn(cr, fplog, "The stochastic dynamics integrator %s is deprecated, since\n"
"it is slower than integrator %s and is slightly less accurate\n"
"with constraints. Use the %s integrator.",
ei_names[inputrec->eI], ei_names[eiSD1], ei_names[eiSD1]);
}
}
/* Check and update the hardware options for internal consistency */
check_and_update_hw_opt_1(hw_opt, cr);
/* Early check for externally set process affinity. */
gmx_check_thread_affinity_set(fplog, cr,
hw_opt, hwinfo->nthreads_hw_avail, FALSE);
#ifdef GMX_THREAD_MPI
if (SIMMASTER(cr))
{
if (cr->npmenodes > 0 && hw_opt->nthreads_tmpi <= 0)
{
gmx_fatal(FARGS, "You need to explicitly specify the number of MPI threads (-ntmpi) when using separate PME ranks");
}
/* Since the master knows the cut-off scheme, update hw_opt for this.
* This is done later for normal MPI and also once more with tMPI
* for all tMPI ranks.
*/
check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme);
/* NOW the threads will be started: */
hw_opt->nthreads_tmpi = get_nthreads_mpi(hwinfo,
hw_opt,
inputrec, mtop,
cr, fplog, bUseGPU);
if (hw_opt->nthreads_tmpi > 1)
{
t_commrec *cr_old = cr;
/* now start the threads. */
cr = mdrunner_start_threads(hw_opt, fplog, cr_old, nfile, fnm,
oenv, bVerbose, bCompact, nstglobalcomm,
ddxyz, dd_node_order, rdd, rconstr,
dddlb_opt, dlb_scale, ddcsx, ddcsy, ddcsz,
nbpu_opt, nstlist_cmdline,
nsteps_cmdline, nstepout, resetstep, nmultisim,
repl_ex_nst, repl_ex_nex, repl_ex_seed, pforce,
cpt_period, max_hours,
Flags);
/* the main thread continues here with a new cr. We don't deallocate
the old cr because other threads may still be reading it. */
if (cr == NULL)
{
gmx_comm("Failed to spawn threads");
}
}
}
#endif
/* END OF CAUTION: cr is now reliable */
/* g_membed initialisation *
* Because we change the mtop, init_membed is called before the init_parallel *
* (in case we ever want to make it run in parallel) */
if (opt2bSet("-membed", nfile, fnm))
{
if (MASTER(cr))
{
fprintf(stderr, "Initializing membed");
}
membed = init_membed(fplog, nfile, fnm, mtop, inputrec, state, cr, &cpt_period);
}
if (PAR(cr))
{
/* now broadcast everything to the non-master nodes/threads: */
init_parallel(cr, inputrec, mtop);
/* The master rank decided on the use of GPUs,
* broadcast this information to all ranks.
*/
gmx_bcast_sim(sizeof(bUseGPU), &bUseGPU, cr);
}
if (fplog != NULL)
{
pr_inputrec(fplog, 0, "Input Parameters", inputrec, FALSE);
fprintf(fplog, "\n");
}
/* now make sure the state is initialized and propagated */
set_state_entries(state, inputrec);
/* A parallel command line option consistency check that we can
only do after any threads have started. */
if (!PAR(cr) &&
(ddxyz[XX] > 1 || ddxyz[YY] > 1 || ddxyz[ZZ] > 1 || cr->npmenodes > 0))
{
gmx_fatal(FARGS,
"The -dd or -npme option request a parallel simulation, "
#ifndef GMX_MPI
"but %s was compiled without threads or MPI enabled"
#else
#ifdef GMX_THREAD_MPI
"but the number of threads (option -nt) is 1"
#else
"but %s was not started through mpirun/mpiexec or only one rank was requested through mpirun/mpiexec"
#endif
#endif
, output_env_get_program_display_name(oenv)
);
}
if (bRerunMD &&
(EI_ENERGY_MINIMIZATION(inputrec->eI) || eiNM == inputrec->eI))
{
gmx_fatal(FARGS, "The .mdp file specified an energy mininization or normal mode algorithm, and these are not compatible with mdrun -rerun");
}
if (can_use_allvsall(inputrec, TRUE, cr, fplog) && DOMAINDECOMP(cr))
{
gmx_fatal(FARGS, "All-vs-all loops do not work with domain decomposition, use a single MPI rank");
}
if (!(EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype)))
{
if (cr->npmenodes > 0)
{
gmx_fatal_collective(FARGS, cr, NULL,
"PME-only ranks are requested, but the system does not use PME for electrostatics or LJ");
}
cr->npmenodes = 0;
}
if (bUseGPU && cr->npmenodes < 0)
{
/* With GPUs we don't automatically use PME-only ranks. PME ranks can
* improve performance with many threads per GPU, since our OpenMP
* scaling is bad, but it's difficult to automate the setup.
*/
cr->npmenodes = 0;
}
#ifdef GMX_FAHCORE
if (MASTER(cr))
{
fcRegisterSteps(inputrec->nsteps, inputrec->init_step);
}
#endif
/* NMR restraints must be initialized before load_checkpoint,
* since with time averaging the history is added to t_state.
* For proper consistency check we therefore need to extend
* t_state here.
* So the PME-only nodes (if present) will also initialize
* the distance restraints.
*/
snew(fcd, 1);
/* This needs to be called before read_checkpoint to extend the state */
init_disres(fplog, mtop, inputrec, cr, fcd, state, repl_ex_nst > 0);
init_orires(fplog, mtop, state->x, inputrec, cr, &(fcd->orires),
state);
if (DEFORM(*inputrec))
{
/* Store the deform reference box before reading the checkpoint */
if (SIMMASTER(cr))
{
copy_mat(state->box, box);
}
if (PAR(cr))
{
gmx_bcast(sizeof(box), box, cr);
}
/* Because we do not have the update struct available yet
* in which the reference values should be stored,
* we store them temporarily in static variables.
* This should be thread safe, since they are only written once
* and with identical values.
*/
tMPI_Thread_mutex_lock(&deform_init_box_mutex);
deform_init_init_step_tpx = inputrec->init_step;
copy_mat(box, deform_init_box_tpx);
tMPI_Thread_mutex_unlock(&deform_init_box_mutex);
}
if (opt2bSet("-cpi", nfile, fnm))
{
/* Check if checkpoint file exists before doing continuation.
* This way we can use identical input options for the first and subsequent runs...
*/
if (gmx_fexist_master(opt2fn_master("-cpi", nfile, fnm, cr), cr) )
{
load_checkpoint(opt2fn_master("-cpi", nfile, fnm, cr), &fplog,
cr, ddxyz,
inputrec, state, &bReadEkin,
(Flags & MD_APPENDFILES),
(Flags & MD_APPENDFILESSET));
if (bReadEkin)
{
Flags |= MD_READ_EKIN;
}
}
}
if (MASTER(cr) && (Flags & MD_APPENDFILES))
{
gmx_log_open(ftp2fn(efLOG, nfile, fnm), cr,
Flags, &fplog);
}
/* override nsteps with value from cmdline */
override_nsteps_cmdline(fplog, nsteps_cmdline, inputrec, cr);
if (SIMMASTER(cr))
{
copy_mat(state->box, box);
}
if (PAR(cr))
{
gmx_bcast(sizeof(box), box, cr);
}
/* Essential dynamics */
if (opt2bSet("-ei", nfile, fnm))
{
/* Open input and output files, allocate space for ED data structure */
ed = ed_open(mtop->natoms, &state->edsamstate, nfile, fnm, Flags, oenv, cr);
}
if (PAR(cr) && !(EI_TPI(inputrec->eI) ||
inputrec->eI == eiNM))
{
cr->dd = init_domain_decomposition(fplog, cr, Flags, ddxyz, rdd, rconstr,
dddlb_opt, dlb_scale,
ddcsx, ddcsy, ddcsz,
mtop, inputrec,
box, state->x,
&ddbox, &npme_major, &npme_minor);
make_dd_communicators(fplog, cr, dd_node_order);
/* Set overallocation to avoid frequent reallocation of arrays */
set_over_alloc_dd(TRUE);
}
else
{
/* PME, if used, is done on all nodes with 1D decomposition */
cr->npmenodes = 0;
cr->duty = (DUTY_PP | DUTY_PME);
npme_major = 1;
npme_minor = 1;
if (inputrec->ePBC == epbcSCREW)
{
gmx_fatal(FARGS,
"pbc=%s is only implemented with domain decomposition",
epbc_names[inputrec->ePBC]);
}
}
if (PAR(cr))
{
/* After possible communicator splitting in make_dd_communicators.
* we can set up the intra/inter node communication.
*/
gmx_setup_nodecomm(fplog, cr);
}
/* Initialize per-physical-node MPI process/thread ID and counters. */
gmx_init_intranode_counters(cr);
#ifdef GMX_MPI
if (MULTISIM(cr))
{
md_print_info(cr, fplog,
"This is simulation %d out of %d running as a composite GROMACS\n"
"multi-simulation job. Setup for this simulation:\n\n",
cr->ms->sim, cr->ms->nsim);
}
md_print_info(cr, fplog, "Using %d MPI %s\n",
cr->nnodes,
#ifdef GMX_THREAD_MPI
cr->nnodes == 1 ? "thread" : "threads"
#else
cr->nnodes == 1 ? "process" : "processes"
#endif
);
fflush(stderr);
#endif
/* Check and update hw_opt for the cut-off scheme */
check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme);
/* Check and update hw_opt for the number of MPI ranks */
check_and_update_hw_opt_3(hw_opt);
gmx_omp_nthreads_init(fplog, cr,
hwinfo->nthreads_hw_avail,
hw_opt->nthreads_omp,
hw_opt->nthreads_omp_pme,
(cr->duty & DUTY_PP) == 0,
inputrec->cutoff_scheme == ecutsVERLET);
#ifndef NDEBUG
if (integrator[inputrec->eI].func != do_tpi &&
inputrec->cutoff_scheme == ecutsVERLET)
{
gmx_feenableexcept();
}
#endif
if (bUseGPU)
{
/* Select GPU id's to use */
gmx_select_gpu_ids(fplog, cr, &hwinfo->gpu_info, bForceUseGPU,
&hw_opt->gpu_opt);
}
else
{
/* Ignore (potentially) manually selected GPUs */
hw_opt->gpu_opt.n_dev_use = 0;
}
/* check consistency across ranks of things like SIMD
* support and number of GPUs selected */
gmx_check_hw_runconf_consistency(fplog, hwinfo, cr, hw_opt, bUseGPU);
/* Now that we know the setup is consistent, check for efficiency */
check_resource_division_efficiency(hwinfo, hw_opt, Flags & MD_NTOMPSET,
cr, fplog);
if (DOMAINDECOMP(cr))
{
/* When we share GPUs over ranks, we need to know this for the DLB */
dd_setup_dlb_resource_sharing(cr, hwinfo, hw_opt);
}
/* getting number of PP/PME threads
PME: env variable should be read only on one node to make sure it is
identical everywhere;
*/
/* TODO nthreads_pp is only used for pinning threads.
* This is a temporary solution until we have a hw topology library.
*/
nthreads_pp = gmx_omp_nthreads_get(emntNonbonded);
nthreads_pme = gmx_omp_nthreads_get(emntPME);
wcycle = wallcycle_init(fplog, resetstep, cr, nthreads_pp, nthreads_pme);
if (PAR(cr))
{
/* Master synchronizes its value of reset_counters with all nodes
* including PME only nodes */
reset_counters = wcycle_get_reset_counters(wcycle);
gmx_bcast_sim(sizeof(reset_counters), &reset_counters, cr);
wcycle_set_reset_counters(wcycle, reset_counters);
}
snew(nrnb, 1);
if (cr->duty & DUTY_PP)
{
bcast_state(cr, state);
/* Initiate forcerecord */
fr = mk_forcerec();
fr->hwinfo = hwinfo;
fr->gpu_opt = &hw_opt->gpu_opt;
init_forcerec(fplog, oenv, fr, fcd, inputrec, mtop, cr, box,
opt2fn("-table", nfile, fnm),
opt2fn("-tabletf", nfile, fnm),
opt2fn("-tablep", nfile, fnm),
opt2fn("-tableb", nfile, fnm),
nbpu_opt,
FALSE,
pforce);
/* version for PCA_NOT_READ_NODE (see md.c) */
/*init_forcerec(fplog,fr,fcd,inputrec,mtop,cr,box,FALSE,
"nofile","nofile","nofile","nofile",FALSE,pforce);
*/
/* Initialize QM-MM */
if (fr->bQMMM)
{
init_QMMMrec(cr, mtop, inputrec, fr);
}
/* Initialize the mdatoms structure.
* mdatoms is not filled with atom data,
* as this can not be done now with domain decomposition.
*/
mdatoms = init_mdatoms(fplog, mtop, inputrec->efep != efepNO);
/* Initialize the virtual site communication */
vsite = init_vsite(mtop, cr, FALSE);
calc_shifts(box, fr->shift_vec);
/* With periodic molecules the charge groups should be whole at start up
* and the virtual sites should not be far from their proper positions.
*/
if (!inputrec->bContinuation && MASTER(cr) &&
!(inputrec->ePBC != epbcNONE && inputrec->bPeriodicMols))
{
/* Make molecules whole at start of run */
if (fr->ePBC != epbcNONE)
{
do_pbc_first_mtop(fplog, inputrec->ePBC, box, mtop, state->x);
}
if (vsite)
{
/* Correct initial vsite positions are required
* for the initial distribution in the domain decomposition
* and for the initial shell prediction.
*/
construct_vsites_mtop(vsite, mtop, state->x);
}
}
if (EEL_PME(fr->eeltype) || EVDW_PME(fr->vdwtype))
{
ewaldcoeff_q = fr->ewaldcoeff_q;
ewaldcoeff_lj = fr->ewaldcoeff_lj;
pmedata = &fr->pmedata;
}
else
{
pmedata = NULL;
}
}
else
{
/* This is a PME only node */
/* We don't need the state */
done_state(state);
ewaldcoeff_q = calc_ewaldcoeff_q(inputrec->rcoulomb, inputrec->ewald_rtol);
ewaldcoeff_lj = calc_ewaldcoeff_lj(inputrec->rvdw, inputrec->ewald_rtol_lj);
snew(pmedata, 1);
}
if (hw_opt->thread_affinity != threadaffOFF)
{
/* Before setting affinity, check whether the affinity has changed
* - which indicates that probably the OpenMP library has changed it
* since we first checked).
*/
gmx_check_thread_affinity_set(fplog, cr,
hw_opt, hwinfo->nthreads_hw_avail, TRUE);
/* Set the CPU affinity */
gmx_set_thread_affinity(fplog, cr, hw_opt, hwinfo);
}
/* Initiate PME if necessary,
* either on all nodes or on dedicated PME nodes only. */
if (EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype))
{
if (mdatoms)
{
nChargePerturbed = mdatoms->nChargePerturbed;
if (EVDW_PME(inputrec->vdwtype))
{
nTypePerturbed = mdatoms->nTypePerturbed;
}
}
if (cr->npmenodes > 0)
{
/* The PME only nodes need to know nChargePerturbed(FEP on Q) and nTypePerturbed(FEP on LJ)*/
gmx_bcast_sim(sizeof(nChargePerturbed), &nChargePerturbed, cr);
gmx_bcast_sim(sizeof(nTypePerturbed), &nTypePerturbed, cr);
}
if (cr->duty & DUTY_PME)
{
status = gmx_pme_init(pmedata, cr, npme_major, npme_minor, inputrec,
mtop ? mtop->natoms : 0, nChargePerturbed, nTypePerturbed,
(Flags & MD_REPRODUCIBLE), nthreads_pme);
if (status != 0)
{
gmx_fatal(FARGS, "Error %d initializing PME", status);
}
}
}
if (integrator[inputrec->eI].func == do_md)
{
/* Turn on signal handling on all nodes */
/*
* (A user signal from the PME nodes (if any)
* is communicated to the PP nodes.
*/
signal_handler_install();
}
if (cr->duty & DUTY_PP)
{
/* Assumes uniform use of the number of OpenMP threads */
walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntDefault));
if (inputrec->bPull)
{
/* Initialize pull code */
inputrec->pull_work =
init_pull(fplog, inputrec->pull, inputrec, nfile, fnm,
mtop, cr, oenv, inputrec->fepvals->init_lambda,
EI_DYNAMICS(inputrec->eI) && MASTER(cr), Flags);
}
if (inputrec->bRot)
{
/* Initialize enforced rotation code */
init_rot(fplog, inputrec, nfile, fnm, cr, state->x, box, mtop, oenv,
bVerbose, Flags);
}
if (inputrec->eSwapCoords != eswapNO)
{
/* Initialize ion swapping code */
init_swapcoords(fplog, bVerbose, inputrec, opt2fn_master("-swap", nfile, fnm, cr),
mtop, state->x, state->box, &state->swapstate, cr, oenv, Flags);
}
constr = init_constraints(fplog, mtop, inputrec, ed, state, cr);
if (DOMAINDECOMP(cr))
{
GMX_RELEASE_ASSERT(fr, "fr was NULL while cr->duty was DUTY_PP");
dd_init_bondeds(fplog, cr->dd, mtop, vsite, inputrec,
Flags & MD_DDBONDCHECK, fr->cginfo_mb);
set_dd_parameters(fplog, cr->dd, dlb_scale, inputrec, &ddbox);
setup_dd_grid(fplog, cr->dd);
}
/* Now do whatever the user wants us to do (how flexible...) */
integrator[inputrec->eI].func(fplog, cr, nfile, fnm,
oenv, bVerbose, bCompact,
nstglobalcomm,
vsite, constr,
nstepout, inputrec, mtop,
fcd, state,
mdatoms, nrnb, wcycle, ed, fr,
repl_ex_nst, repl_ex_nex, repl_ex_seed,
membed,
cpt_period, max_hours,
imdport,
Flags,
walltime_accounting);
if (inputrec->bPull)
{
finish_pull(inputrec->pull_work);
}
if (inputrec->bRot)
{
finish_rot(inputrec->rot);
}
}
else
{
GMX_RELEASE_ASSERT(pmedata, "pmedata was NULL while cr->duty was not DUTY_PP");
/* do PME only */
walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntPME));
gmx_pmeonly(*pmedata, cr, nrnb, wcycle, walltime_accounting, ewaldcoeff_q, ewaldcoeff_lj, inputrec);
}
wallcycle_stop(wcycle, ewcRUN);
/* Finish up, write some stuff
* if rerunMD, don't write last frame again
*/
finish_run(fplog, cr,
inputrec, nrnb, wcycle, walltime_accounting,
fr ? fr->nbv : NULL,
EI_DYNAMICS(inputrec->eI) && !MULTISIM(cr));
/* Free GPU memory and context */
free_gpu_resources(fr, cr, &hwinfo->gpu_info, fr ? fr->gpu_opt : NULL);
if (opt2bSet("-membed", nfile, fnm))
{
sfree(membed);
}
gmx_hardware_info_free(hwinfo);
/* Does what it says */
print_date_and_time(fplog, cr->nodeid, "Finished mdrun", gmx_gettime());
walltime_accounting_destroy(walltime_accounting);
/* PLUMED */
if(plumedswitch){
plumed_finalize(plumedmain);
}
/* END PLUMED */
/* Close logfile already here if we were appending to it */
if (MASTER(cr) && (Flags & MD_APPENDFILES))
{
gmx_log_close(fplog);
}
rc = (int)gmx_get_stop_condition();
done_ed(&ed);
#ifdef GMX_THREAD_MPI
/* we need to join all threads. The sub-threads join when they
exit this function, but the master thread needs to be told to
wait for that. */
if (PAR(cr) && MASTER(cr))
{
tMPI_Finalize();
}
#endif
return rc;
}
