static void track_disp_hdr_cb(const route_head *track) { cur_info = &route_info[route_idx]; cur_info->prev_wpt = NULL; cur_info->total = 0; current_trk = (route_head *)track; if (track->rte_waypt_ct <= 0) return; if (!gtxt_flags.track_header_written) { gtxt_flags.track_header_written = 1; gbfprintf(fout, "\r\n\r\nHeader\t%s\r\n", headers[track_header]); } print_string("\r\nTrack\t%s\t", current_trk->rte_name ? current_trk->rte_name : ""); print_date_and_time(cur_info->start, 0); print_date_and_time(cur_info->time, 1); print_distance(cur_info->length, 0, 1, 0); print_speed(&cur_info->length, &cur_info->time); print_string("%s", (track->rte_url != NULL) ? track->rte_url : ""); gbfprintf(fout, "\r\n\r\nHeader\t%s\r\n\r\n", headers[trkpt_header]); }
static void track_disp_wpt_cb(const waypoint *wpt) { waypoint *prev = cur_info->prev_wpt; time_t delta; double dist, depth; gbfprintf(fout, "Trackpoint\t"); print_position(wpt); print_date_and_time(wpt->creation_time, 0); if IS_VALID_ALT(wpt->altitude) print_distance(wpt->altitude, 1, 0, 0); gbfprintf(fout, "\t"); depth = WAYPT_GET(wpt, depth, unknown_alt); if (depth != unknown_alt) print_distance(depth, 1, 0, 1); if (prev != NULL) { float temp; gbfprintf(fout, "\t"); delta = wpt->creation_time - prev->creation_time; temp = WAYPT_GET(wpt, temperature, -999); if (temp != -999) print_temperature(temp); gbfprintf(fout, "\t"); dist = waypt_distance_ex(prev, wpt); print_distance(dist, 0, 1, 0); print_date_and_time(delta, 1); print_speed(&dist, &delta); print_course(prev, wpt); } gbfprintf(fout, "\r\n"); cur_info->prev_wpt = (waypoint *)wpt; }
double do_md_openmm(FILE *fplog,t_commrec *cr,int nfile,const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,gmx_bool bCompact, int nstglobalcomm, gmx_vsite_t *vsite,gmx_constr_t constr, int stepout,t_inputrec *ir, gmx_mtop_t *top_global, t_fcdata *fcd, t_state *state_global, t_mdatoms *mdatoms, t_nrnb *nrnb,gmx_wallcycle_t wcycle, gmx_edsam_t ed,t_forcerec *fr, int repl_ex_nst,int repl_ex_seed, real cpt_period,real max_hours, const char *deviceOptions, unsigned long Flags, gmx_runtime_t *runtime) { gmx_mdoutf_t *outf; gmx_large_int_t step,step_rel; double run_time; double t,t0,lam0; gmx_bool bSimAnn, bFirstStep,bStateFromTPX,bLastStep,bStartingFromCpt; gmx_bool bInitStep=TRUE; gmx_bool do_ene,do_log, do_verbose, bX,bV,bF,bCPT; tensor force_vir,shake_vir,total_vir,pres; int i,m; int mdof_flags; rvec mu_tot; t_vcm *vcm; int nchkpt=1; gmx_localtop_t *top; t_mdebin *mdebin=NULL; t_state *state=NULL; rvec *f_global=NULL; int n_xtc=-1; rvec *x_xtc=NULL; gmx_enerdata_t *enerd; rvec *f=NULL; gmx_global_stat_t gstat; gmx_update_t upd=NULL; t_graph *graph=NULL; globsig_t gs; gmx_groups_t *groups; gmx_ekindata_t *ekind, *ekind_save; gmx_bool bAppend; int a0,a1; matrix lastbox; real reset_counters=0,reset_counters_now=0; char sbuf[STEPSTRSIZE],sbuf2[STEPSTRSIZE]; int handled_stop_condition=gmx_stop_cond_none; const char *ommOptions = NULL; void *openmmData; bAppend = (Flags & MD_APPENDFILES); check_ir_old_tpx_versions(cr,fplog,ir,top_global); groups = &top_global->groups; /* Initial values */ init_md(fplog,cr,ir,oenv,&t,&t0,&state_global->lambda,&lam0, nrnb,top_global,&upd, nfile,fnm,&outf,&mdebin, force_vir,shake_vir,mu_tot,&bSimAnn,&vcm,state_global,Flags); clear_mat(total_vir); clear_mat(pres); /* Energy terms and groups */ snew(enerd,1); init_enerdata(top_global->groups.grps[egcENER].nr,ir->n_flambda,enerd); snew(f,top_global->natoms); /* Kinetic energy data */ snew(ekind,1); init_ekindata(fplog,top_global,&(ir->opts),ekind); /* needed for iteration of constraints */ snew(ekind_save,1); init_ekindata(fplog,top_global,&(ir->opts),ekind_save); /* Copy the cos acceleration to the groups struct */ ekind->cosacc.cos_accel = ir->cos_accel; gstat = global_stat_init(ir); debug_gmx(); { double io = compute_io(ir,top_global->natoms,groups,mdebin->ebin->nener,1); if ((io > 2000) && MASTER(cr)) fprintf(stderr, "\nWARNING: This run will generate roughly %.0f Mb of data\n\n", io); } top = gmx_mtop_generate_local_top(top_global,ir); a0 = 0; a1 = top_global->natoms; state = partdec_init_local_state(cr,state_global); f_global = f; atoms2md(top_global,ir,0,NULL,a0,a1-a0,mdatoms); if (vsite) { set_vsite_top(vsite,top,mdatoms,cr); } if (ir->ePBC != epbcNONE && !ir->bPeriodicMols) { graph = mk_graph(fplog,&(top->idef),0,top_global->natoms,FALSE,FALSE); } update_mdatoms(mdatoms,state->lambda); if (deviceOptions[0]=='\0') { /* empty options, which should default to OpenMM in this build */ ommOptions=deviceOptions; } else { if (gmx_strncasecmp(deviceOptions,"OpenMM",6)!=0) { gmx_fatal(FARGS, "This Gromacs version currently only works with OpenMM. Use -device \"OpenMM:<options>\""); } else { ommOptions=strchr(deviceOptions,':'); if (NULL!=ommOptions) { /* Increase the pointer to skip the colon */ ommOptions++; } } } openmmData = openmm_init(fplog, ommOptions, ir, top_global, top, mdatoms, fr, state); please_cite(fplog,"Friedrichs2009"); if (MASTER(cr)) { /* Update mdebin with energy history if appending to output files */ if ( Flags & MD_APPENDFILES ) { restore_energyhistory_from_state(mdebin,&state_global->enerhist); } /* Set the initial energy history in state to zero by updating once */ update_energyhistory(&state_global->enerhist,mdebin); } if (constr) { set_constraints(constr,top,ir,mdatoms,cr); } if (!ir->bContinuation) { if (mdatoms->cFREEZE && (state->flags & (1<<estV))) { /* Set the velocities of frozen particles to zero */ for (i=mdatoms->start; i<mdatoms->start+mdatoms->homenr; i++) { for (m=0; m<DIM; m++) { if (ir->opts.nFreeze[mdatoms->cFREEZE[i]][m]) { state->v[i][m] = 0; } } } } if (constr) { /* Constrain the initial coordinates and velocities */ do_constrain_first(fplog,constr,ir,mdatoms,state,f, graph,cr,nrnb,fr,top,shake_vir); } if (vsite) { /* Construct the virtual sites for the initial configuration */ construct_vsites(fplog,vsite,state->x,nrnb,ir->delta_t,NULL, top->idef.iparams,top->idef.il, fr->ePBC,fr->bMolPBC,graph,cr,state->box); } } debug_gmx(); if (MASTER(cr)) { char tbuf[20]; fprintf(fplog,"Initial temperature: %g K\n",enerd->term[F_TEMP]); fprintf(stderr,"starting mdrun '%s'\n", *(top_global->name)); if (ir->nsteps >= 0) { sprintf(tbuf,"%8.1f",(ir->init_step+ir->nsteps)*ir->delta_t); } else { sprintf(tbuf,"%s","infinite"); } if (ir->init_step > 0) { fprintf(stderr,"%s steps, %s ps (continuing from step %s, %8.1f ps).\n", gmx_step_str(ir->init_step+ir->nsteps,sbuf),tbuf, gmx_step_str(ir->init_step,sbuf2), ir->init_step*ir->delta_t); } else { fprintf(stderr,"%s steps, %s ps.\n", gmx_step_str(ir->nsteps,sbuf),tbuf); } } fprintf(fplog,"\n"); /* Set and write start time */ runtime_start(runtime); print_date_and_time(fplog,cr->nodeid,"Started mdrun",runtime); wallcycle_start(wcycle,ewcRUN); if (fplog) fprintf(fplog,"\n"); /* safest point to do file checkpointing is here. More general point would be immediately before integrator call */ debug_gmx(); /*********************************************************** * * Loop over MD steps * ************************************************************/ /* loop over MD steps or if rerunMD to end of input trajectory */ bFirstStep = TRUE; /* Skip the first Nose-Hoover integration when we get the state from tpx */ bStateFromTPX = !opt2bSet("-cpi",nfile,fnm); bInitStep = bFirstStep && bStateFromTPX; bStartingFromCpt = (Flags & MD_STARTFROMCPT) && bInitStep; bLastStep = FALSE; init_global_signals(&gs,cr,ir,repl_ex_nst); step = ir->init_step; step_rel = 0; while (!bLastStep) { wallcycle_start(wcycle,ewcSTEP); GMX_MPE_LOG(ev_timestep1); bLastStep = (step_rel == ir->nsteps); t = t0 + step*ir->delta_t; if (gs.set[eglsSTOPCOND] != 0) { bLastStep = TRUE; } do_log = do_per_step(step,ir->nstlog) || bFirstStep || bLastStep; do_verbose = bVerbose && (step % stepout == 0 || bFirstStep || bLastStep); if (MASTER(cr) && do_log) { print_ebin_header(fplog,step,t,state->lambda); } clear_mat(force_vir); GMX_MPE_LOG(ev_timestep2); /* We write a checkpoint at this MD step when: * either when we signalled through gs (in OpenMM NS works different), * or at the last step (but not when we do not want confout), * but never at the first step. */ bCPT = ((gs.set[eglsCHKPT] || (bLastStep && (Flags & MD_CONFOUT))) && step > ir->init_step ); if (bCPT) { gs.set[eglsCHKPT] = 0; } /* Now we have the energies and forces corresponding to the * coordinates at time t. We must output all of this before * the update. * for RerunMD t is read from input trajectory */ GMX_MPE_LOG(ev_output_start); mdof_flags = 0; if (do_per_step(step,ir->nstxout)) { mdof_flags |= MDOF_X; } if (do_per_step(step,ir->nstvout)) { mdof_flags |= MDOF_V; } if (do_per_step(step,ir->nstfout)) { mdof_flags |= MDOF_F; } if (do_per_step(step,ir->nstxtcout)) { mdof_flags |= MDOF_XTC; } if (bCPT) { mdof_flags |= MDOF_CPT; }; do_ene = (do_per_step(step,ir->nstenergy) || bLastStep); if (mdof_flags != 0 || do_ene || do_log) { wallcycle_start(wcycle,ewcTRAJ); bF = (mdof_flags & MDOF_F); bX = (mdof_flags & (MDOF_X | MDOF_XTC | MDOF_CPT)); bV = (mdof_flags & (MDOF_V | MDOF_CPT)); openmm_copy_state(openmmData, state, &t, f, enerd, bX, bV, bF, do_ene); upd_mdebin(mdebin, FALSE,TRUE, t,mdatoms->tmass,enerd,state,lastbox, shake_vir,force_vir,total_vir,pres, ekind,mu_tot,constr); print_ebin(outf->fp_ene,do_ene,FALSE,FALSE,do_log?fplog:NULL, step,t, eprNORMAL,bCompact,mdebin,fcd,groups,&(ir->opts)); write_traj(fplog,cr,outf,mdof_flags,top_global, step,t,state,state_global,f,f_global,&n_xtc,&x_xtc); if (bCPT) { nchkpt++; bCPT = FALSE; } debug_gmx(); if (bLastStep && step_rel == ir->nsteps && (Flags & MD_CONFOUT) && MASTER(cr)) { /* x and v have been collected in write_traj, * because a checkpoint file will always be written * at the last step. */ fprintf(stderr,"\nWriting final coordinates.\n"); if (ir->ePBC != epbcNONE && !ir->bPeriodicMols) { /* Make molecules whole only for confout writing */ do_pbc_mtop(fplog,ir->ePBC,state->box,top_global,state_global->x); } write_sto_conf_mtop(ftp2fn(efSTO,nfile,fnm), *top_global->name,top_global, state_global->x,state_global->v, ir->ePBC,state->box); debug_gmx(); } wallcycle_stop(wcycle,ewcTRAJ); } GMX_MPE_LOG(ev_output_finish); /* Determine the wallclock run time up till now */ run_time = gmx_gettime() - (double)runtime->real; /* Check whether everything is still allright */ if (((int)gmx_get_stop_condition() > handled_stop_condition) #ifdef GMX_THREADS && MASTER(cr) #endif ) { /* this is just make gs.sig compatible with the hack of sending signals around by MPI_Reduce with together with other floats */ /* NOTE: this only works for serial code. For code that allows MPI nodes to propagate their condition, see kernel/md.c*/ if ( gmx_get_stop_condition() == gmx_stop_cond_next_ns ) gs.set[eglsSTOPCOND]=1; if ( gmx_get_stop_condition() == gmx_stop_cond_next ) gs.set[eglsSTOPCOND]=1; /* < 0 means stop at next step, > 0 means stop at next NS step */ if (fplog) { fprintf(fplog, "\n\nReceived the %s signal, stopping at the next %sstep\n\n", gmx_get_signal_name(), gs.sig[eglsSTOPCOND]==1 ? "NS " : ""); fflush(fplog); } fprintf(stderr, "\n\nReceived the %s signal, stopping at the next %sstep\n\n", gmx_get_signal_name(), gs.sig[eglsSTOPCOND]==1 ? "NS " : ""); fflush(stderr); handled_stop_condition=(int)gmx_get_stop_condition(); } else if (MASTER(cr) && (max_hours > 0 && run_time > max_hours*60.0*60.0*0.99) && gs.set[eglsSTOPCOND] == 0) { /* Signal to terminate the run */ gs.set[eglsSTOPCOND] = 1; if (fplog) { fprintf(fplog,"\nStep %s: Run time exceeded %.3f hours, will terminate the run\n",gmx_step_str(step,sbuf),max_hours*0.99); } fprintf(stderr, "\nStep %s: Run time exceeded %.3f hours, will terminate the run\n",gmx_step_str(step,sbuf),max_hours*0.99); } /* checkpoints */ if (MASTER(cr) && (cpt_period >= 0 && (cpt_period == 0 || run_time >= nchkpt*cpt_period*60.0)) && gs.set[eglsCHKPT] == 0) { gs.set[eglsCHKPT] = 1; } /* Time for performance */ if (((step % stepout) == 0) || bLastStep) { runtime_upd_proc(runtime); } if (do_per_step(step,ir->nstlog)) { if (fflush(fplog) != 0) { gmx_fatal(FARGS,"Cannot flush logfile - maybe you are out of quota?"); } } /* Remaining runtime */ if (MULTIMASTER(cr) && (do_verbose || gmx_got_usr_signal() )) { print_time(stderr,runtime,step,ir,cr); } bFirstStep = FALSE; bInitStep = FALSE; bStartingFromCpt = FALSE; step++; step_rel++; openmm_take_one_step(openmmData); } /* End of main MD loop */ debug_gmx(); /* Stop the time */ runtime_end(runtime); if (MASTER(cr)) { if (ir->nstcalcenergy > 0) { print_ebin(outf->fp_ene,FALSE,FALSE,FALSE,fplog,step,t, eprAVER,FALSE,mdebin,fcd,groups,&(ir->opts)); } } openmm_cleanup(fplog, openmmData); done_mdoutf(outf); debug_gmx(); runtime->nsteps_done = step_rel; return 0; }
static void write_waypt(const waypoint *wpt) { unsigned char wpt_class; garmin_fs_p gmsd; char *wpt_type; char *dspl_mode; const char *country; double x; int i, icon, dynamic; char *icon_descr; gmsd = GMSD_FIND(wpt); i = GMSD_GET(display, 0); if (i > GT_DISPLAY_MODE_MAX) i = 0; dspl_mode = gt_display_mode_names[i]; wpt_class = GMSD_GET(wpt_class, 0); if (wpt_class <= gt_waypt_class_map_line) wpt_type = gt_waypt_class_names[wpt_class]; else wpt_type = gt_waypt_class_names[0]; gbfprintf(fout, "Waypoint\t%s\t", (wpt->shortname) ? wpt->shortname : ""); if (wpt_class <= gt_waypt_class_airport_ndb) { char *temp = wpt->notes; if (temp == NULL) { if (wpt->description && (strcmp(wpt->description, wpt->shortname) != 0)) temp = wpt->description; else temp = ""; } print_string("%s\t", temp); } else gbfprintf(fout, "\t"); gbfprintf(fout, "%s\t", wpt_type); print_position(wpt); if IS_VALID_ALT(wpt->altitude) print_distance(wpt->altitude, 1, 0, 0); gbfprintf(fout, "\t"); x = WAYPT_GET(wpt, depth, unknown_alt); if (x != unknown_alt) print_distance(x, 1, 0, 1); gbfprintf(fout, "\t"); x = WAYPT_GET(wpt, proximity, unknown_alt); if (x != unknown_alt) print_distance(x, 0, 0, 0); gbfprintf(fout, "\t"); x = WAYPT_GET(wpt, temperature, -999); if (x != -999) print_temperature(x); gbfprintf(fout, "\t%s\t", dspl_mode); gbfprintf(fout, "Unknown\t"); /* Color is fixed: Unknown */ icon = GMSD_GET(icon, -1); if (icon == -1) { icon = gt_find_icon_number_from_desc(wpt->icon_descr, GDB); } icon_descr = gt_find_desc_from_icon_number(icon, GDB, &dynamic); print_string("%s\t", icon_descr); if (dynamic) xfree(icon_descr); print_string("%s\t", GMSD_GET(facility, "")); print_string("%s\t", GMSD_GET(city, "")); print_string("%s\t", GMSD_GET(state, "")); country = gt_get_icao_country(GMSD_GET(cc, "")); print_string("%s\t", (country != NULL) ? country : ""); print_date_and_time(wpt->creation_time, 0); print_string("%s\t", wpt->url ? wpt->url : ""); print_categories(GMSD_GET(category, 0)); gbfprintf(fout, "\r\n"); }
double do_tpi(FILE *fplog,t_commrec *cr, int nfile, const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,gmx_bool bCompact, int nstglobalcomm, gmx_vsite_t *vsite,gmx_constr_t constr, int stepout, t_inputrec *inputrec, gmx_mtop_t *top_global,t_fcdata *fcd, t_state *state, t_mdatoms *mdatoms, t_nrnb *nrnb,gmx_wallcycle_t wcycle, gmx_edsam_t ed, t_forcerec *fr, int repl_ex_nst,int repl_ex_seed, real cpt_period,real max_hours, const char *deviceOptions, unsigned long Flags, gmx_runtime_t *runtime) { const char *TPI="Test Particle Insertion"; gmx_localtop_t *top; gmx_groups_t *groups; gmx_enerdata_t *enerd; rvec *f; real lambda,t,temp,beta,drmax,epot; double embU,sum_embU,*sum_UgembU,V,V_all,VembU_all; t_trxstatus *status; t_trxframe rerun_fr; gmx_bool bDispCorr,bCharge,bRFExcl,bNotLastFrame,bStateChanged,bNS,bOurStep; tensor force_vir,shake_vir,vir,pres; int cg_tp,a_tp0,a_tp1,ngid,gid_tp,nener,e; rvec *x_mol; rvec mu_tot,x_init,dx,x_tp; int nnodes,frame,nsteps,step; int i,start,end; gmx_rng_t tpi_rand; FILE *fp_tpi=NULL; char *ptr,*dump_pdb,**leg,str[STRLEN],str2[STRLEN]; double dbl,dump_ener; gmx_bool bCavity; int nat_cavity=0,d; real *mass_cavity=NULL,mass_tot; int nbin; double invbinw,*bin,refvolshift,logV,bUlogV; real dvdl,prescorr,enercorr,dvdlcorr; gmx_bool bEnergyOutOfBounds; const char *tpid_leg[2]={"direct","reweighted"}; /* Since there is no upper limit to the insertion energies, * we need to set an upper limit for the distribution output. */ real bU_bin_limit = 50; real bU_logV_bin_limit = bU_bin_limit + 10; nnodes = cr->nnodes; top = gmx_mtop_generate_local_top(top_global,inputrec); groups = &top_global->groups; bCavity = (inputrec->eI == eiTPIC); if (bCavity) { ptr = getenv("GMX_TPIC_MASSES"); if (ptr == NULL) { nat_cavity = 1; } else { /* Read (multiple) masses from env var GMX_TPIC_MASSES, * The center of mass of the last atoms is then used for TPIC. */ nat_cavity = 0; while (sscanf(ptr,"%lf%n",&dbl,&i) > 0) { srenew(mass_cavity,nat_cavity+1); mass_cavity[nat_cavity] = dbl; fprintf(fplog,"mass[%d] = %f\n", nat_cavity+1,mass_cavity[nat_cavity]); nat_cavity++; ptr += i; } if (nat_cavity == 0) gmx_fatal(FARGS,"Found %d masses in GMX_TPIC_MASSES",nat_cavity); } } /* init_em(fplog,TPI,inputrec,&lambda,nrnb,mu_tot, state->box,fr,mdatoms,top,cr,nfile,fnm,NULL,NULL);*/ /* We never need full pbc for TPI */ fr->ePBC = epbcXYZ; /* Determine the temperature for the Boltzmann weighting */ temp = inputrec->opts.ref_t[0]; if (fplog) { for(i=1; (i<inputrec->opts.ngtc); i++) { if (inputrec->opts.ref_t[i] != temp) { fprintf(fplog,"\nWARNING: The temperatures of the different temperature coupling groups are not identical\n\n"); fprintf(stderr,"\nWARNING: The temperatures of the different temperature coupling groups are not identical\n\n"); } } fprintf(fplog, "\n The temperature for test particle insertion is %.3f K\n\n", temp); } beta = 1.0/(BOLTZ*temp); /* Number of insertions per frame */ nsteps = inputrec->nsteps; /* Use the same neighborlist with more insertions points * in a sphere of radius drmax around the initial point */ /* This should be a proper mdp parameter */ drmax = inputrec->rtpi; /* An environment variable can be set to dump all configurations * to pdb with an insertion energy <= this value. */ dump_pdb = getenv("GMX_TPI_DUMP"); dump_ener = 0; if (dump_pdb) sscanf(dump_pdb,"%lf",&dump_ener); atoms2md(top_global,inputrec,0,NULL,0,top_global->natoms,mdatoms); update_mdatoms(mdatoms,inputrec->init_lambda); snew(enerd,1); init_enerdata(groups->grps[egcENER].nr,inputrec->n_flambda,enerd); snew(f,top_global->natoms); /* Print to log file */ runtime_start(runtime); print_date_and_time(fplog,cr->nodeid, "Started Test Particle Insertion",runtime); wallcycle_start(wcycle,ewcRUN); /* The last charge group is the group to be inserted */ cg_tp = top->cgs.nr - 1; a_tp0 = top->cgs.index[cg_tp]; a_tp1 = top->cgs.index[cg_tp+1]; if (debug) fprintf(debug,"TPI cg %d, atoms %d-%d\n",cg_tp,a_tp0,a_tp1); if (a_tp1 - a_tp0 > 1 && (inputrec->rlist < inputrec->rcoulomb || inputrec->rlist < inputrec->rvdw)) gmx_fatal(FARGS,"Can not do TPI for multi-atom molecule with a twin-range cut-off"); snew(x_mol,a_tp1-a_tp0); bDispCorr = (inputrec->eDispCorr != edispcNO); bCharge = FALSE; for(i=a_tp0; i<a_tp1; i++) { /* Copy the coordinates of the molecule to be insterted */ copy_rvec(state->x[i],x_mol[i-a_tp0]); /* Check if we need to print electrostatic energies */ bCharge |= (mdatoms->chargeA[i] != 0 || (mdatoms->chargeB && mdatoms->chargeB[i] != 0)); } bRFExcl = (bCharge && EEL_RF(fr->eeltype) && fr->eeltype!=eelRF_NEC); calc_cgcm(fplog,cg_tp,cg_tp+1,&(top->cgs),state->x,fr->cg_cm); if (bCavity) { if (norm(fr->cg_cm[cg_tp]) > 0.5*inputrec->rlist && fplog) { fprintf(fplog, "WARNING: Your TPI molecule is not centered at 0,0,0\n"); fprintf(stderr,"WARNING: Your TPI molecule is not centered at 0,0,0\n"); } } else { /* Center the molecule to be inserted at zero */ for(i=0; i<a_tp1-a_tp0; i++) rvec_dec(x_mol[i],fr->cg_cm[cg_tp]); } if (fplog) { fprintf(fplog,"\nWill insert %d atoms %s partial charges\n", a_tp1-a_tp0,bCharge ? "with" : "without"); fprintf(fplog,"\nWill insert %d times in each frame of %s\n", nsteps,opt2fn("-rerun",nfile,fnm)); } if (!bCavity) { if (inputrec->nstlist > 1) { if (drmax==0 && a_tp1-a_tp0==1) { gmx_fatal(FARGS,"Re-using the neighborlist %d times for insertions of a single atom in a sphere of radius %f does not make sense",inputrec->nstlist,drmax); } if (fplog) { fprintf(fplog,"Will use the same neighborlist for %d insertions in a sphere of radius %f\n",inputrec->nstlist,drmax); } } } else { if (fplog) { fprintf(fplog,"Will insert randomly in a sphere of radius %f around the center of the cavity\n",drmax); } } ngid = groups->grps[egcENER].nr; gid_tp = GET_CGINFO_GID(fr->cginfo[cg_tp]); nener = 1 + ngid; if (bDispCorr) nener += 1; if (bCharge) { nener += ngid; if (bRFExcl) nener += 1; if (EEL_FULL(fr->eeltype)) nener += 1; } snew(sum_UgembU,nener); /* Initialize random generator */ tpi_rand = gmx_rng_init(inputrec->ld_seed); if (MASTER(cr)) { fp_tpi = xvgropen(opt2fn("-tpi",nfile,fnm), "TPI energies","Time (ps)", "(kJ mol\\S-1\\N) / (nm\\S3\\N)",oenv); xvgr_subtitle(fp_tpi,"f. are averages over one frame",oenv); snew(leg,4+nener); e = 0; sprintf(str,"-kT log(<Ve\\S-\\betaU\\N>/<V>)"); leg[e++] = strdup(str); sprintf(str,"f. -kT log<e\\S-\\betaU\\N>"); leg[e++] = strdup(str); sprintf(str,"f. <e\\S-\\betaU\\N>"); leg[e++] = strdup(str); sprintf(str,"f. V"); leg[e++] = strdup(str); sprintf(str,"f. <Ue\\S-\\betaU\\N>"); leg[e++] = strdup(str); for(i=0; i<ngid; i++) { sprintf(str,"f. <U\\sVdW %s\\Ne\\S-\\betaU\\N>", *(groups->grpname[groups->grps[egcENER].nm_ind[i]])); leg[e++] = strdup(str); } if (bDispCorr) { sprintf(str,"f. <U\\sdisp c\\Ne\\S-\\betaU\\N>"); leg[e++] = strdup(str); } if (bCharge) { for(i=0; i<ngid; i++) { sprintf(str,"f. <U\\sCoul %s\\Ne\\S-\\betaU\\N>", *(groups->grpname[groups->grps[egcENER].nm_ind[i]])); leg[e++] = strdup(str); } if (bRFExcl) { sprintf(str,"f. <U\\sRF excl\\Ne\\S-\\betaU\\N>"); leg[e++] = strdup(str); } if (EEL_FULL(fr->eeltype)) { sprintf(str,"f. <U\\sCoul recip\\Ne\\S-\\betaU\\N>"); leg[e++] = strdup(str); } } xvgr_legend(fp_tpi,4+nener,(const char**)leg,oenv); for(i=0; i<4+nener; i++) sfree(leg[i]); sfree(leg); } clear_rvec(x_init); V_all = 0; VembU_all = 0; invbinw = 10; nbin = 10; snew(bin,nbin); bNotLastFrame = read_first_frame(oenv,&status,opt2fn("-rerun",nfile,fnm), &rerun_fr,TRX_NEED_X); frame = 0; if (rerun_fr.natoms - (bCavity ? nat_cavity : 0) != mdatoms->nr - (a_tp1 - a_tp0)) gmx_fatal(FARGS,"Number of atoms in trajectory (%d)%s " "is not equal the number in the run input file (%d) " "minus the number of atoms to insert (%d)\n", rerun_fr.natoms,bCavity ? " minus one" : "", mdatoms->nr,a_tp1-a_tp0); refvolshift = log(det(rerun_fr.box)); #if ( defined(GMX_IA32_SSE) || defined(GMX_X86_64_SSE) || defined(GMX_X86_64_SSE2) ) /* Make sure we don't detect SSE overflow generated before this point */ gmx_mm_check_and_reset_overflow(); #endif while (bNotLastFrame) { lambda = rerun_fr.lambda; t = rerun_fr.time; sum_embU = 0; for(e=0; e<nener; e++) { sum_UgembU[e] = 0; } /* Copy the coordinates from the input trajectory */ for(i=0; i<rerun_fr.natoms; i++) { copy_rvec(rerun_fr.x[i],state->x[i]); } V = det(rerun_fr.box); logV = log(V); bStateChanged = TRUE; bNS = TRUE; for(step=0; step<nsteps; step++) { /* In parallel all nodes generate all random configurations. * In that way the result is identical to a single cpu tpi run. */ if (!bCavity) { /* Random insertion in the whole volume */ bNS = (step % inputrec->nstlist == 0); if (bNS) { /* Generate a random position in the box */ x_init[XX] = gmx_rng_uniform_real(tpi_rand)*state->box[XX][XX]; x_init[YY] = gmx_rng_uniform_real(tpi_rand)*state->box[YY][YY]; x_init[ZZ] = gmx_rng_uniform_real(tpi_rand)*state->box[ZZ][ZZ]; } if (inputrec->nstlist == 1) { copy_rvec(x_init,x_tp); } else { /* Generate coordinates within |dx|=drmax of x_init */ do { dx[XX] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax; dx[YY] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax; dx[ZZ] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax; } while (norm2(dx) > drmax*drmax); rvec_add(x_init,dx,x_tp); } } else { /* Random insertion around a cavity location * given by the last coordinate of the trajectory. */ if (step == 0) { if (nat_cavity == 1) { /* Copy the location of the cavity */ copy_rvec(rerun_fr.x[rerun_fr.natoms-1],x_init); } else { /* Determine the center of mass of the last molecule */ clear_rvec(x_init); mass_tot = 0; for(i=0; i<nat_cavity; i++) { for(d=0; d<DIM; d++) { x_init[d] += mass_cavity[i]*rerun_fr.x[rerun_fr.natoms-nat_cavity+i][d]; } mass_tot += mass_cavity[i]; } for(d=0; d<DIM; d++) { x_init[d] /= mass_tot; } } } /* Generate coordinates within |dx|=drmax of x_init */ do { dx[XX] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax; dx[YY] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax; dx[ZZ] = (2*gmx_rng_uniform_real(tpi_rand) - 1)*drmax; } while (norm2(dx) > drmax*drmax); rvec_add(x_init,dx,x_tp); } if (a_tp1 - a_tp0 == 1) { /* Insert a single atom, just copy the insertion location */ copy_rvec(x_tp,state->x[a_tp0]); } else { /* Copy the coordinates from the top file */ for(i=a_tp0; i<a_tp1; i++) { copy_rvec(x_mol[i-a_tp0],state->x[i]); } /* Rotate the molecule randomly */ rotate_conf(a_tp1-a_tp0,state->x+a_tp0,NULL, 2*M_PI*gmx_rng_uniform_real(tpi_rand), 2*M_PI*gmx_rng_uniform_real(tpi_rand), 2*M_PI*gmx_rng_uniform_real(tpi_rand)); /* Shift to the insertion location */ for(i=a_tp0; i<a_tp1; i++) { rvec_inc(state->x[i],x_tp); } } /* Check if this insertion belongs to this node */ bOurStep = TRUE; if (PAR(cr)) { switch (inputrec->eI) { case eiTPI: bOurStep = ((step / inputrec->nstlist) % nnodes == cr->nodeid); break; case eiTPIC: bOurStep = (step % nnodes == cr->nodeid); break; default: gmx_fatal(FARGS,"Unknown integrator %s",ei_names[inputrec->eI]); } } if (bOurStep) { /* Clear some matrix variables */ clear_mat(force_vir); clear_mat(shake_vir); clear_mat(vir); clear_mat(pres); /* Set the charge group center of mass of the test particle */ copy_rvec(x_init,fr->cg_cm[top->cgs.nr-1]); /* Calc energy (no forces) on new positions. * Since we only need the intermolecular energy * and the RF exclusion terms of the inserted molecule occur * within a single charge group we can pass NULL for the graph. * This also avoids shifts that would move charge groups * out of the box. * * Some checks above ensure than we can not have * twin-range interactions together with nstlist > 1, * therefore we do not need to remember the LR energies. */ /* Make do_force do a single node force calculation */ cr->nnodes = 1; do_force(fplog,cr,inputrec, step,nrnb,wcycle,top,top_global,&top_global->groups, rerun_fr.box,state->x,&state->hist, f,force_vir,mdatoms,enerd,fcd, lambda,NULL,fr,NULL,mu_tot,t,NULL,NULL,FALSE, GMX_FORCE_NONBONDED | (bNS ? GMX_FORCE_NS | GMX_FORCE_DOLR : 0) | (bStateChanged ? GMX_FORCE_STATECHANGED : 0)); cr->nnodes = nnodes; bStateChanged = FALSE; bNS = FALSE; /* Calculate long range corrections to pressure and energy */ calc_dispcorr(fplog,inputrec,fr,step,top_global->natoms,rerun_fr.box, lambda,pres,vir,&prescorr,&enercorr,&dvdlcorr); /* figure out how to rearrange the next 4 lines MRS 8/4/2009 */ enerd->term[F_DISPCORR] = enercorr; enerd->term[F_EPOT] += enercorr; enerd->term[F_PRES] += prescorr; enerd->term[F_DVDL] += dvdlcorr; epot = enerd->term[F_EPOT]; bEnergyOutOfBounds = FALSE; #if ( defined(GMX_IA32_SSE) || defined(GMX_X86_64_SSE) || defined(GMX_X86_64_SSE2) ) /* With SSE the energy can overflow, check for this */ if (gmx_mm_check_and_reset_overflow()) { if (debug) { fprintf(debug,"Found an SSE overflow, assuming the energy is out of bounds\n"); } bEnergyOutOfBounds = TRUE; } #endif /* If the compiler doesn't optimize this check away * we catch the NAN energies. * The epot>GMX_REAL_MAX check catches inf values, * which should nicely result in embU=0 through the exp below, * but it does not hurt to check anyhow. */ /* Non-bonded Interaction usually diverge at r=0. * With tabulated interaction functions the first few entries * should be capped in a consistent fashion between * repulsion, dispersion and Coulomb to avoid accidental * negative values in the total energy. * The table generation code in tables.c does this. * With user tbales the user should take care of this. */ if (epot != epot || epot > GMX_REAL_MAX) { bEnergyOutOfBounds = TRUE; } if (bEnergyOutOfBounds) { if (debug) { fprintf(debug,"\n time %.3f, step %d: non-finite energy %f, using exp(-bU)=0\n",t,step,epot); } embU = 0; } else { embU = exp(-beta*epot); sum_embU += embU; /* Determine the weighted energy contributions of each energy group */ e = 0; sum_UgembU[e++] += epot*embU; if (fr->bBHAM) { for(i=0; i<ngid; i++) { sum_UgembU[e++] += (enerd->grpp.ener[egBHAMSR][GID(i,gid_tp,ngid)] + enerd->grpp.ener[egBHAMLR][GID(i,gid_tp,ngid)])*embU; } } else { for(i=0; i<ngid; i++) { sum_UgembU[e++] += (enerd->grpp.ener[egLJSR][GID(i,gid_tp,ngid)] + enerd->grpp.ener[egLJLR][GID(i,gid_tp,ngid)])*embU; } } if (bDispCorr) { sum_UgembU[e++] += enerd->term[F_DISPCORR]*embU; } if (bCharge) { for(i=0; i<ngid; i++) { sum_UgembU[e++] += (enerd->grpp.ener[egCOULSR][GID(i,gid_tp,ngid)] + enerd->grpp.ener[egCOULLR][GID(i,gid_tp,ngid)])*embU; } if (bRFExcl) { sum_UgembU[e++] += enerd->term[F_RF_EXCL]*embU; } if (EEL_FULL(fr->eeltype)) { sum_UgembU[e++] += enerd->term[F_COUL_RECIP]*embU; } } } if (embU == 0 || beta*epot > bU_bin_limit) { bin[0]++; } else { i = (int)((bU_logV_bin_limit - (beta*epot - logV + refvolshift))*invbinw + 0.5); if (i < 0) { i = 0; } if (i >= nbin) { realloc_bins(&bin,&nbin,i+10); } bin[i]++; } if(fr->adress_do_drift ||fr->adress_icor == eAdressICThermoForce ){ sum_UgembU[e++] += enerd->term[F_ADR_DELTU]*embU; } if (debug) { fprintf(debug,"TPI %7d %12.5e %12.5f %12.5f %12.5f\n", step,epot,x_tp[XX],x_tp[YY],x_tp[ZZ]); } if (dump_pdb && epot <= dump_ener) { sprintf(str,"t%g_step%d.pdb",t,step); sprintf(str2,"t: %f step %d ener: %f",t,step,epot); write_sto_conf_mtop(str,str2,top_global,state->x,state->v, inputrec->ePBC,state->box); } } } if (PAR(cr)) { /* When running in parallel sum the energies over the processes */ gmx_sumd(1, &sum_embU, cr); gmx_sumd(nener,sum_UgembU,cr); } frame++; V_all += V; VembU_all += V*sum_embU/nsteps; if (fp_tpi) { if (bVerbose || frame%10==0 || frame<10) { fprintf(stderr,"mu %10.3e <mu> %10.3e\n", -log(sum_embU/nsteps)/beta,-log(VembU_all/V_all)/beta); } fprintf(fp_tpi,"%10.3f %12.5e %12.5e %12.5e %12.5e", t, VembU_all==0 ? 20/beta : -log(VembU_all/V_all)/beta, sum_embU==0 ? 20/beta : -log(sum_embU/nsteps)/beta, sum_embU/nsteps,V); for(e=0; e<nener; e++) { fprintf(fp_tpi," %12.5e",sum_UgembU[e]/nsteps); } fprintf(fp_tpi,"\n"); fflush(fp_tpi); } bNotLastFrame = read_next_frame(oenv, status,&rerun_fr); } /* End of the loop */ runtime_end(runtime); close_trj(status); if (fp_tpi != NULL) { gmx_fio_fclose(fp_tpi); } if (fplog != NULL) { fprintf(fplog,"\n"); fprintf(fplog," <V> = %12.5e nm^3\n",V_all/frame); fprintf(fplog," <mu> = %12.5e kJ/mol\n",-log(VembU_all/V_all)/beta); } /* Write the Boltzmann factor histogram */ if (PAR(cr)) { /* When running in parallel sum the bins over the processes */ i = nbin; global_max(cr,&i); realloc_bins(&bin,&nbin,i); gmx_sumd(nbin,bin,cr); } if (MASTER(cr)) { fp_tpi = xvgropen(opt2fn("-tpid",nfile,fnm), "TPI energy distribution", "\\betaU - log(V/<V>)","count",oenv); sprintf(str,"number \\betaU > %g: %9.3e",bU_bin_limit,bin[0]); xvgr_subtitle(fp_tpi,str,oenv); xvgr_legend(fp_tpi,2,(const char **)tpid_leg,oenv); for(i=nbin-1; i>0; i--) { bUlogV = -i/invbinw + bU_logV_bin_limit - refvolshift + log(V_all/frame); fprintf(fp_tpi,"%6.2f %10d %12.5e\n", bUlogV, (int)(bin[i]+0.5), bin[i]*exp(-bUlogV)*V_all/VembU_all); } gmx_fio_fclose(fp_tpi); } sfree(bin); sfree(sum_UgembU); runtime->nsteps_done = frame*inputrec->nsteps; return 0; }
int mdrunner(gmx_hw_opt_t *hw_opt, FILE *fplog, t_commrec *cr, int nfile, const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose, gmx_bool bCompact, int nstglobalcomm, ivec ddxyz, int dd_node_order, real rdd, real rconstr, const char *dddlb_opt, real dlb_scale, const char *ddcsx, const char *ddcsy, const char *ddcsz, const char *nbpu_opt, int nstlist_cmdline, gmx_int64_t nsteps_cmdline, int nstepout, int resetstep, int gmx_unused nmultisim, int repl_ex_nst, int repl_ex_nex, int repl_ex_seed, real pforce, real cpt_period, real max_hours, int imdport, unsigned long Flags) { gmx_bool bForceUseGPU, bTryUseGPU, bRerunMD; t_inputrec *inputrec; t_state *state = NULL; matrix box; gmx_ddbox_t ddbox = {0}; int npme_major, npme_minor; t_nrnb *nrnb; gmx_mtop_t *mtop = NULL; t_mdatoms *mdatoms = NULL; t_forcerec *fr = NULL; t_fcdata *fcd = NULL; real ewaldcoeff_q = 0; real ewaldcoeff_lj = 0; struct gmx_pme_t **pmedata = NULL; gmx_vsite_t *vsite = NULL; gmx_constr_t constr; int nChargePerturbed = -1, nTypePerturbed = 0, status; gmx_wallcycle_t wcycle; gmx_bool bReadEkin; gmx_walltime_accounting_t walltime_accounting = NULL; int rc; gmx_int64_t reset_counters; gmx_edsam_t ed = NULL; int nthreads_pme = 1; int nthreads_pp = 1; gmx_membed_t membed = NULL; gmx_hw_info_t *hwinfo = NULL; /* The master rank decides early on bUseGPU and broadcasts this later */ gmx_bool bUseGPU = FALSE; /* CAUTION: threads may be started later on in this function, so cr doesn't reflect the final parallel state right now */ snew(inputrec, 1); snew(mtop, 1); if (Flags & MD_APPENDFILES) { fplog = NULL; } bRerunMD = (Flags & MD_RERUN); bForceUseGPU = (strncmp(nbpu_opt, "gpu", 3) == 0); bTryUseGPU = (strncmp(nbpu_opt, "auto", 4) == 0) || bForceUseGPU; /* Detect hardware, gather information. This is an operation that is * global for this process (MPI rank). */ hwinfo = gmx_detect_hardware(fplog, cr, bTryUseGPU); gmx_print_detected_hardware(fplog, cr, hwinfo); if (fplog != NULL) { /* Print references after all software/hardware printing */ please_cite(fplog, "Abraham2015"); please_cite(fplog, "Pall2015"); please_cite(fplog, "Pronk2013"); please_cite(fplog, "Hess2008b"); please_cite(fplog, "Spoel2005a"); please_cite(fplog, "Lindahl2001a"); please_cite(fplog, "Berendsen95a"); } snew(state, 1); if (SIMMASTER(cr)) { /* Read (nearly) all data required for the simulation */ read_tpx_state(ftp2fn(efTPR, nfile, fnm), inputrec, state, NULL, mtop); if (inputrec->cutoff_scheme == ecutsVERLET) { /* Here the master rank decides if all ranks will use GPUs */ bUseGPU = (hwinfo->gpu_info.n_dev_compatible > 0 || getenv("GMX_EMULATE_GPU") != NULL); /* TODO add GPU kernels for this and replace this check by: * (bUseGPU && (ir->vdwtype == evdwPME && * ir->ljpme_combination_rule == eljpmeLB)) * update the message text and the content of nbnxn_acceleration_supported. */ if (bUseGPU && !nbnxn_gpu_acceleration_supported(fplog, cr, inputrec, bRerunMD)) { /* Fallback message printed by nbnxn_acceleration_supported */ if (bForceUseGPU) { gmx_fatal(FARGS, "GPU acceleration requested, but not supported with the given input settings"); } bUseGPU = FALSE; } prepare_verlet_scheme(fplog, cr, inputrec, nstlist_cmdline, mtop, state->box, bUseGPU); } else { if (nstlist_cmdline > 0) { gmx_fatal(FARGS, "Can not set nstlist with the group cut-off scheme"); } if (hwinfo->gpu_info.n_dev_compatible > 0) { md_print_warn(cr, fplog, "NOTE: GPU(s) found, but the current simulation can not use GPUs\n" " To use a GPU, set the mdp option: cutoff-scheme = Verlet\n"); } if (bForceUseGPU) { gmx_fatal(FARGS, "GPU requested, but can't be used without cutoff-scheme=Verlet"); } #ifdef GMX_TARGET_BGQ md_print_warn(cr, fplog, "NOTE: There is no SIMD implementation of the group scheme kernels on\n" " BlueGene/Q. You will observe better performance from using the\n" " Verlet cut-off scheme.\n"); #endif } if (inputrec->eI == eiSD2) { md_print_warn(cr, fplog, "The stochastic dynamics integrator %s is deprecated, since\n" "it is slower than integrator %s and is slightly less accurate\n" "with constraints. Use the %s integrator.", ei_names[inputrec->eI], ei_names[eiSD1], ei_names[eiSD1]); } } /* Check and update the hardware options for internal consistency */ check_and_update_hw_opt_1(hw_opt, cr); /* Early check for externally set process affinity. */ gmx_check_thread_affinity_set(fplog, cr, hw_opt, hwinfo->nthreads_hw_avail, FALSE); #ifdef GMX_THREAD_MPI if (SIMMASTER(cr)) { if (cr->npmenodes > 0 && hw_opt->nthreads_tmpi <= 0) { gmx_fatal(FARGS, "You need to explicitly specify the number of MPI threads (-ntmpi) when using separate PME ranks"); } /* Since the master knows the cut-off scheme, update hw_opt for this. * This is done later for normal MPI and also once more with tMPI * for all tMPI ranks. */ check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme); /* NOW the threads will be started: */ hw_opt->nthreads_tmpi = get_nthreads_mpi(hwinfo, hw_opt, inputrec, mtop, cr, fplog, bUseGPU); if (hw_opt->nthreads_tmpi > 1) { t_commrec *cr_old = cr; /* now start the threads. */ cr = mdrunner_start_threads(hw_opt, fplog, cr_old, nfile, fnm, oenv, bVerbose, bCompact, nstglobalcomm, ddxyz, dd_node_order, rdd, rconstr, dddlb_opt, dlb_scale, ddcsx, ddcsy, ddcsz, nbpu_opt, nstlist_cmdline, nsteps_cmdline, nstepout, resetstep, nmultisim, repl_ex_nst, repl_ex_nex, repl_ex_seed, pforce, cpt_period, max_hours, Flags); /* the main thread continues here with a new cr. We don't deallocate the old cr because other threads may still be reading it. */ if (cr == NULL) { gmx_comm("Failed to spawn threads"); } } } #endif /* END OF CAUTION: cr is now reliable */ /* g_membed initialisation * * Because we change the mtop, init_membed is called before the init_parallel * * (in case we ever want to make it run in parallel) */ if (opt2bSet("-membed", nfile, fnm)) { if (MASTER(cr)) { fprintf(stderr, "Initializing membed"); } membed = init_membed(fplog, nfile, fnm, mtop, inputrec, state, cr, &cpt_period); } if (PAR(cr)) { /* now broadcast everything to the non-master nodes/threads: */ init_parallel(cr, inputrec, mtop); /* The master rank decided on the use of GPUs, * broadcast this information to all ranks. */ gmx_bcast_sim(sizeof(bUseGPU), &bUseGPU, cr); } if (fplog != NULL) { pr_inputrec(fplog, 0, "Input Parameters", inputrec, FALSE); fprintf(fplog, "\n"); } /* now make sure the state is initialized and propagated */ set_state_entries(state, inputrec); /* A parallel command line option consistency check that we can only do after any threads have started. */ if (!PAR(cr) && (ddxyz[XX] > 1 || ddxyz[YY] > 1 || ddxyz[ZZ] > 1 || cr->npmenodes > 0)) { gmx_fatal(FARGS, "The -dd or -npme option request a parallel simulation, " #ifndef GMX_MPI "but %s was compiled without threads or MPI enabled" #else #ifdef GMX_THREAD_MPI "but the number of threads (option -nt) is 1" #else "but %s was not started through mpirun/mpiexec or only one rank was requested through mpirun/mpiexec" #endif #endif , output_env_get_program_display_name(oenv) ); } if (bRerunMD && (EI_ENERGY_MINIMIZATION(inputrec->eI) || eiNM == inputrec->eI)) { gmx_fatal(FARGS, "The .mdp file specified an energy mininization or normal mode algorithm, and these are not compatible with mdrun -rerun"); } if (can_use_allvsall(inputrec, TRUE, cr, fplog) && DOMAINDECOMP(cr)) { gmx_fatal(FARGS, "All-vs-all loops do not work with domain decomposition, use a single MPI rank"); } if (!(EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype))) { if (cr->npmenodes > 0) { gmx_fatal_collective(FARGS, cr, NULL, "PME-only ranks are requested, but the system does not use PME for electrostatics or LJ"); } cr->npmenodes = 0; } if (bUseGPU && cr->npmenodes < 0) { /* With GPUs we don't automatically use PME-only ranks. PME ranks can * improve performance with many threads per GPU, since our OpenMP * scaling is bad, but it's difficult to automate the setup. */ cr->npmenodes = 0; } #ifdef GMX_FAHCORE if (MASTER(cr)) { fcRegisterSteps(inputrec->nsteps, inputrec->init_step); } #endif /* NMR restraints must be initialized before load_checkpoint, * since with time averaging the history is added to t_state. * For proper consistency check we therefore need to extend * t_state here. * So the PME-only nodes (if present) will also initialize * the distance restraints. */ snew(fcd, 1); /* This needs to be called before read_checkpoint to extend the state */ init_disres(fplog, mtop, inputrec, cr, fcd, state, repl_ex_nst > 0); init_orires(fplog, mtop, state->x, inputrec, cr, &(fcd->orires), state); if (DEFORM(*inputrec)) { /* Store the deform reference box before reading the checkpoint */ if (SIMMASTER(cr)) { copy_mat(state->box, box); } if (PAR(cr)) { gmx_bcast(sizeof(box), box, cr); } /* Because we do not have the update struct available yet * in which the reference values should be stored, * we store them temporarily in static variables. * This should be thread safe, since they are only written once * and with identical values. */ tMPI_Thread_mutex_lock(&deform_init_box_mutex); deform_init_init_step_tpx = inputrec->init_step; copy_mat(box, deform_init_box_tpx); tMPI_Thread_mutex_unlock(&deform_init_box_mutex); } if (opt2bSet("-cpi", nfile, fnm)) { /* Check if checkpoint file exists before doing continuation. * This way we can use identical input options for the first and subsequent runs... */ if (gmx_fexist_master(opt2fn_master("-cpi", nfile, fnm, cr), cr) ) { load_checkpoint(opt2fn_master("-cpi", nfile, fnm, cr), &fplog, cr, ddxyz, inputrec, state, &bReadEkin, (Flags & MD_APPENDFILES), (Flags & MD_APPENDFILESSET)); if (bReadEkin) { Flags |= MD_READ_EKIN; } } } if (MASTER(cr) && (Flags & MD_APPENDFILES)) { gmx_log_open(ftp2fn(efLOG, nfile, fnm), cr, Flags, &fplog); } /* override nsteps with value from cmdline */ override_nsteps_cmdline(fplog, nsteps_cmdline, inputrec, cr); if (SIMMASTER(cr)) { copy_mat(state->box, box); } if (PAR(cr)) { gmx_bcast(sizeof(box), box, cr); } /* Essential dynamics */ if (opt2bSet("-ei", nfile, fnm)) { /* Open input and output files, allocate space for ED data structure */ ed = ed_open(mtop->natoms, &state->edsamstate, nfile, fnm, Flags, oenv, cr); } if (PAR(cr) && !(EI_TPI(inputrec->eI) || inputrec->eI == eiNM)) { cr->dd = init_domain_decomposition(fplog, cr, Flags, ddxyz, rdd, rconstr, dddlb_opt, dlb_scale, ddcsx, ddcsy, ddcsz, mtop, inputrec, box, state->x, &ddbox, &npme_major, &npme_minor); make_dd_communicators(fplog, cr, dd_node_order); /* Set overallocation to avoid frequent reallocation of arrays */ set_over_alloc_dd(TRUE); } else { /* PME, if used, is done on all nodes with 1D decomposition */ cr->npmenodes = 0; cr->duty = (DUTY_PP | DUTY_PME); npme_major = 1; npme_minor = 1; if (inputrec->ePBC == epbcSCREW) { gmx_fatal(FARGS, "pbc=%s is only implemented with domain decomposition", epbc_names[inputrec->ePBC]); } } if (PAR(cr)) { /* After possible communicator splitting in make_dd_communicators. * we can set up the intra/inter node communication. */ gmx_setup_nodecomm(fplog, cr); } /* Initialize per-physical-node MPI process/thread ID and counters. */ gmx_init_intranode_counters(cr); #ifdef GMX_MPI if (MULTISIM(cr)) { md_print_info(cr, fplog, "This is simulation %d out of %d running as a composite GROMACS\n" "multi-simulation job. Setup for this simulation:\n\n", cr->ms->sim, cr->ms->nsim); } md_print_info(cr, fplog, "Using %d MPI %s\n", cr->nnodes, #ifdef GMX_THREAD_MPI cr->nnodes == 1 ? "thread" : "threads" #else cr->nnodes == 1 ? "process" : "processes" #endif ); fflush(stderr); #endif /* Check and update hw_opt for the cut-off scheme */ check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme); /* Check and update hw_opt for the number of MPI ranks */ check_and_update_hw_opt_3(hw_opt); gmx_omp_nthreads_init(fplog, cr, hwinfo->nthreads_hw_avail, hw_opt->nthreads_omp, hw_opt->nthreads_omp_pme, (cr->duty & DUTY_PP) == 0, inputrec->cutoff_scheme == ecutsVERLET); #ifndef NDEBUG if (integrator[inputrec->eI].func != do_tpi && inputrec->cutoff_scheme == ecutsVERLET) { gmx_feenableexcept(); } #endif if (bUseGPU) { /* Select GPU id's to use */ gmx_select_gpu_ids(fplog, cr, &hwinfo->gpu_info, bForceUseGPU, &hw_opt->gpu_opt); } else { /* Ignore (potentially) manually selected GPUs */ hw_opt->gpu_opt.n_dev_use = 0; } /* check consistency across ranks of things like SIMD * support and number of GPUs selected */ gmx_check_hw_runconf_consistency(fplog, hwinfo, cr, hw_opt, bUseGPU); /* Now that we know the setup is consistent, check for efficiency */ check_resource_division_efficiency(hwinfo, hw_opt, Flags & MD_NTOMPSET, cr, fplog); if (DOMAINDECOMP(cr)) { /* When we share GPUs over ranks, we need to know this for the DLB */ dd_setup_dlb_resource_sharing(cr, hwinfo, hw_opt); } /* getting number of PP/PME threads PME: env variable should be read only on one node to make sure it is identical everywhere; */ /* TODO nthreads_pp is only used for pinning threads. * This is a temporary solution until we have a hw topology library. */ nthreads_pp = gmx_omp_nthreads_get(emntNonbonded); nthreads_pme = gmx_omp_nthreads_get(emntPME); wcycle = wallcycle_init(fplog, resetstep, cr, nthreads_pp, nthreads_pme); if (PAR(cr)) { /* Master synchronizes its value of reset_counters with all nodes * including PME only nodes */ reset_counters = wcycle_get_reset_counters(wcycle); gmx_bcast_sim(sizeof(reset_counters), &reset_counters, cr); wcycle_set_reset_counters(wcycle, reset_counters); } snew(nrnb, 1); if (cr->duty & DUTY_PP) { bcast_state(cr, state); /* Initiate forcerecord */ fr = mk_forcerec(); fr->hwinfo = hwinfo; fr->gpu_opt = &hw_opt->gpu_opt; init_forcerec(fplog, oenv, fr, fcd, inputrec, mtop, cr, box, opt2fn("-table", nfile, fnm), opt2fn("-tabletf", nfile, fnm), opt2fn("-tablep", nfile, fnm), opt2fn("-tableb", nfile, fnm), nbpu_opt, FALSE, pforce); /* version for PCA_NOT_READ_NODE (see md.c) */ /*init_forcerec(fplog,fr,fcd,inputrec,mtop,cr,box,FALSE, "nofile","nofile","nofile","nofile",FALSE,pforce); */ /* Initialize QM-MM */ if (fr->bQMMM) { init_QMMMrec(cr, mtop, inputrec, fr); } /* Initialize the mdatoms structure. * mdatoms is not filled with atom data, * as this can not be done now with domain decomposition. */ mdatoms = init_mdatoms(fplog, mtop, inputrec->efep != efepNO); /* Initialize the virtual site communication */ vsite = init_vsite(mtop, cr, FALSE); calc_shifts(box, fr->shift_vec); /* With periodic molecules the charge groups should be whole at start up * and the virtual sites should not be far from their proper positions. */ if (!inputrec->bContinuation && MASTER(cr) && !(inputrec->ePBC != epbcNONE && inputrec->bPeriodicMols)) { /* Make molecules whole at start of run */ if (fr->ePBC != epbcNONE) { do_pbc_first_mtop(fplog, inputrec->ePBC, box, mtop, state->x); } if (vsite) { /* Correct initial vsite positions are required * for the initial distribution in the domain decomposition * and for the initial shell prediction. */ construct_vsites_mtop(vsite, mtop, state->x); } } if (EEL_PME(fr->eeltype) || EVDW_PME(fr->vdwtype)) { ewaldcoeff_q = fr->ewaldcoeff_q; ewaldcoeff_lj = fr->ewaldcoeff_lj; pmedata = &fr->pmedata; } else { pmedata = NULL; } } else { /* This is a PME only node */ /* We don't need the state */ done_state(state); ewaldcoeff_q = calc_ewaldcoeff_q(inputrec->rcoulomb, inputrec->ewald_rtol); ewaldcoeff_lj = calc_ewaldcoeff_lj(inputrec->rvdw, inputrec->ewald_rtol_lj); snew(pmedata, 1); } if (hw_opt->thread_affinity != threadaffOFF) { /* Before setting affinity, check whether the affinity has changed * - which indicates that probably the OpenMP library has changed it * since we first checked). */ gmx_check_thread_affinity_set(fplog, cr, hw_opt, hwinfo->nthreads_hw_avail, TRUE); /* Set the CPU affinity */ gmx_set_thread_affinity(fplog, cr, hw_opt, hwinfo); } /* Initiate PME if necessary, * either on all nodes or on dedicated PME nodes only. */ if (EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype)) { if (mdatoms) { nChargePerturbed = mdatoms->nChargePerturbed; if (EVDW_PME(inputrec->vdwtype)) { nTypePerturbed = mdatoms->nTypePerturbed; } } if (cr->npmenodes > 0) { /* The PME only nodes need to know nChargePerturbed(FEP on Q) and nTypePerturbed(FEP on LJ)*/ gmx_bcast_sim(sizeof(nChargePerturbed), &nChargePerturbed, cr); gmx_bcast_sim(sizeof(nTypePerturbed), &nTypePerturbed, cr); } if (cr->duty & DUTY_PME) { status = gmx_pme_init(pmedata, cr, npme_major, npme_minor, inputrec, mtop ? mtop->natoms : 0, nChargePerturbed, nTypePerturbed, (Flags & MD_REPRODUCIBLE), nthreads_pme); if (status != 0) { gmx_fatal(FARGS, "Error %d initializing PME", status); } } } if (integrator[inputrec->eI].func == do_md) { /* Turn on signal handling on all nodes */ /* * (A user signal from the PME nodes (if any) * is communicated to the PP nodes. */ signal_handler_install(); } if (cr->duty & DUTY_PP) { /* Assumes uniform use of the number of OpenMP threads */ walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntDefault)); if (inputrec->bPull) { /* Initialize pull code */ inputrec->pull_work = init_pull(fplog, inputrec->pull, inputrec, nfile, fnm, mtop, cr, oenv, inputrec->fepvals->init_lambda, EI_DYNAMICS(inputrec->eI) && MASTER(cr), Flags); } if (inputrec->bRot) { /* Initialize enforced rotation code */ init_rot(fplog, inputrec, nfile, fnm, cr, state->x, box, mtop, oenv, bVerbose, Flags); } if (inputrec->eSwapCoords != eswapNO) { /* Initialize ion swapping code */ init_swapcoords(fplog, bVerbose, inputrec, opt2fn_master("-swap", nfile, fnm, cr), mtop, state->x, state->box, &state->swapstate, cr, oenv, Flags); } constr = init_constraints(fplog, mtop, inputrec, ed, state, cr); if (DOMAINDECOMP(cr)) { GMX_RELEASE_ASSERT(fr, "fr was NULL while cr->duty was DUTY_PP"); dd_init_bondeds(fplog, cr->dd, mtop, vsite, inputrec, Flags & MD_DDBONDCHECK, fr->cginfo_mb); set_dd_parameters(fplog, cr->dd, dlb_scale, inputrec, &ddbox); setup_dd_grid(fplog, cr->dd); } /* Now do whatever the user wants us to do (how flexible...) */ integrator[inputrec->eI].func(fplog, cr, nfile, fnm, oenv, bVerbose, bCompact, nstglobalcomm, vsite, constr, nstepout, inputrec, mtop, fcd, state, mdatoms, nrnb, wcycle, ed, fr, repl_ex_nst, repl_ex_nex, repl_ex_seed, membed, cpt_period, max_hours, imdport, Flags, walltime_accounting); if (inputrec->bPull) { finish_pull(inputrec->pull_work); } if (inputrec->bRot) { finish_rot(inputrec->rot); } } else { GMX_RELEASE_ASSERT(pmedata, "pmedata was NULL while cr->duty was not DUTY_PP"); /* do PME only */ walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntPME)); gmx_pmeonly(*pmedata, cr, nrnb, wcycle, walltime_accounting, ewaldcoeff_q, ewaldcoeff_lj, inputrec); } wallcycle_stop(wcycle, ewcRUN); /* Finish up, write some stuff * if rerunMD, don't write last frame again */ finish_run(fplog, cr, inputrec, nrnb, wcycle, walltime_accounting, fr ? fr->nbv : NULL, EI_DYNAMICS(inputrec->eI) && !MULTISIM(cr)); /* Free GPU memory and context */ free_gpu_resources(fr, cr, &hwinfo->gpu_info, fr ? fr->gpu_opt : NULL); if (opt2bSet("-membed", nfile, fnm)) { sfree(membed); } gmx_hardware_info_free(hwinfo); /* Does what it says */ print_date_and_time(fplog, cr->nodeid, "Finished mdrun", gmx_gettime()); walltime_accounting_destroy(walltime_accounting); /* PLUMED */ if(plumedswitch){ plumed_finalize(plumedmain); } /* END PLUMED */ /* Close logfile already here if we were appending to it */ if (MASTER(cr) && (Flags & MD_APPENDFILES)) { gmx_log_close(fplog); } rc = (int)gmx_get_stop_condition(); done_ed(&ed); #ifdef GMX_THREAD_MPI /* we need to join all threads. The sub-threads join when they exit this function, but the master thread needs to be told to wait for that. */ if (PAR(cr) && MASTER(cr)) { tMPI_Finalize(); } #endif return rc; }