void
restore_ekinstate_from_state(t_commrec *cr,
gmx_ekindata_t *ekind,ekinstate_t *ekinstate)
{
int i,n;
if (MASTER(cr)) {
for(i=0;i<ekinstate->ekinh_n;i++) {
copy_mat(ekinstate->ekinh[i],ekind->tcstat[i].ekinh);
}
ekind->dekindl = ekinstate->dekindl;
ekind->cosacc.mvcos = ekinstate->mvcos;
n = ekinstate->ekinh_n;
}
if (PAR(cr)) {
gmx_bcast(sizeof(n),&n,cr);
for(i=0;i<n;i++) {
gmx_bcast(DIM*DIM*sizeof(ekind->tcstat[i].ekinh[0][0]),
ekind->tcstat[i].ekinh[0],cr);
}
gmx_bcast(sizeof(ekind->dekindl),&ekind->dekindl,cr);
gmx_bcast(sizeof(ekind->cosacc.mvcos),&ekind->cosacc.mvcos,cr);
}
}
static void comm_args(const t_commrec *cr, int *argc, char ***argv)
{
int i, len;
if (PAR(cr))
{
gmx_bcast(sizeof(*argc), argc, cr);
}
if (!MASTER(cr))
{
snew(*argv, *argc+1);
}
if (debug)
{
fprintf(debug, "NODEID=%d argc=%d\n", cr->nodeid, *argc);
}
for (i = 0; (i < *argc); i++)
{
if (MASTER(cr))
{
len = strlen((*argv)[i])+1;
}
gmx_bcast(sizeof(len), &len, cr);
if (!MASTER(cr))
{
snew((*argv)[i], len);
}
/*gmx_bcast(len*sizeof((*argv)[i][0]),(*argv)[i],cr);*/
gmx_bcast(len*sizeof(char), (*argv)[i], cr);
}
debug_gmx();
}
std::unique_ptr<BoxDeformation>
prepareBoxDeformation(const matrix &initialBox,
t_commrec *cr,
const t_inputrec &inputrec)
{
if (!inputrecDeform(&inputrec))
{
return nullptr;
}
if (!EI_DYNAMICS(inputrec.eI))
{
GMX_THROW(NotImplementedError("Box deformation is only supported with dynamical integrators"));
}
matrix box;
// Only the rank that read the tpr has the global state, and thus
// the initial box, so we pass that around.
if (SIMMASTER(cr))
{
copy_mat(initialBox, box);
}
if (PAR(cr))
{
gmx_bcast(sizeof(box), box, cr);
}
return compat::make_unique<BoxDeformation>(inputrec.delta_t,
inputrec.init_step,
inputrec.deform,
box);
}
int multisim_nstsimsync(const t_commrec *cr,
const t_inputrec *ir, int repl_ex_nst)
{
int nmin;
if (MASTER(cr))
{
nmin = INT_MAX;
nmin = multisim_min(cr->ms, nmin, ir->nstlist);
nmin = multisim_min(cr->ms, nmin, ir->nstcalcenergy);
nmin = multisim_min(cr->ms, nmin, repl_ex_nst);
if (nmin == INT_MAX)
{
gmx_fatal(FARGS, "Can not find an appropriate interval for inter-simulation communication, since nstlist, nstcalcenergy and -replex are all <= 0");
}
/* Avoid inter-simulation communication at every (second) step */
if (nmin <= 2)
{
nmin = 10;
}
}
gmx_bcast(sizeof(int), &nmin, cr);
return nmin;
}
static void comm_args(const t_commrec *cr,int *argc,char ***argv)
{
int i,len;
if ((cr) && PAR(cr))
gmx_bcast(sizeof(*argc),argc,cr);
if (!MASTER(cr))
snew(*argv,*argc+1);
fprintf(stderr,"NODEID=%d argc=%d\n",cr->nodeid,*argc);
for(i=0; (i<*argc); i++) {
if (MASTER(cr))
len = strlen((*argv)[i])+1;
gmx_bcast(sizeof(len),&len,cr);
if (!MASTER(cr))
snew((*argv)[i],len);
gmx_bcast(len*sizeof((*argv)[i][0]),(*argv)[i],cr);
}
debug_gmx();
}
void set_ddbox_cr(t_commrec *cr, ivec *dd_nc,
t_inputrec *ir, matrix box, t_block *cgs, rvec *x,
gmx_ddbox_t *ddbox)
{
if (MASTER(cr))
{
low_set_ddbox(ir, dd_nc, box, TRUE, cgs->nr, cgs, x, NULL, ddbox);
}
gmx_bcast(sizeof(gmx_ddbox_t), ddbox, cr);
}
gmx_bool gmx_fexist_master(const char *fname, t_commrec *cr)
{
gmx_bool bExist;
if (SIMMASTER(cr))
{
bExist = gmx_fexist(fname);
}
if (PAR(cr))
{
gmx_bcast(sizeof(bExist),&bExist,cr);
}
return bExist;
}
void rerun_parallel_comm(t_commrec *cr, t_trxframe *fr,
gmx_bool *bNotLastFrame)
{
rvec *xp, *vp;
if (MASTER(cr) && !*bNotLastFrame)
{
fr->natoms = -1;
}
xp = fr->x;
vp = fr->v;
gmx_bcast(sizeof(*fr), fr, cr);
fr->x = xp;
fr->v = vp;
*bNotLastFrame = (fr->natoms >= 0);
}
/* check which of the multisim simulations has the shortest number of
steps and return that number of nsteps */
gmx_int64_t get_multisim_nsteps(const t_commrec *cr,
gmx_int64_t nsteps)
{
gmx_int64_t steps_out;
if (MASTER(cr))
{
gmx_int64_t *buf;
int s;
snew(buf, cr->ms->nsim);
buf[cr->ms->sim] = nsteps;
gmx_sumli_sim(cr->ms->nsim, buf, cr->ms);
steps_out = -1;
for (s = 0; s < cr->ms->nsim; s++)
{
/* find the smallest positive number */
if (buf[s] >= 0 && ((steps_out < 0) || (buf[s] < steps_out)) )
{
steps_out = buf[s];
}
}
sfree(buf);
/* if we're the limiting simulation, don't do anything */
if (steps_out >= 0 && steps_out < nsteps)
{
char strbuf[255];
snprintf(strbuf, 255, "Will stop simulation %%d after %s steps (another simulation will end then).\n", "%" GMX_PRId64);
fprintf(stderr, strbuf, cr->ms->sim, steps_out);
}
}
/* broadcast to non-masters */
gmx_bcast(sizeof(gmx_int64_t), &steps_out, cr);
return steps_out;
}
void compute_globals(FILE *fplog, gmx_global_stat_t gstat, t_commrec *cr, t_inputrec *ir,
t_forcerec *fr, gmx_ekindata_t *ekind,
t_state *state, t_state *state_global, t_mdatoms *mdatoms,
t_nrnb *nrnb, t_vcm *vcm, gmx_wallcycle_t wcycle,
gmx_enerdata_t *enerd, tensor force_vir, tensor shake_vir, tensor total_vir,
tensor pres, rvec mu_tot, gmx_constr_t constr,
globsig_t *gs, gmx_bool bInterSimGS,
matrix box, gmx_mtop_t *top_global,
gmx_bool *bSumEkinhOld, int flags)
{
int i, gsi;
real gs_buf[eglsNR];
tensor corr_vir, corr_pres;
gmx_bool bEner, bPres, bTemp;
gmx_bool bStopCM, bGStat,
bReadEkin, bEkinAveVel, bScaleEkin, bConstrain;
real prescorr, enercorr, dvdlcorr, dvdl_ekin;
/* translate CGLO flags to gmx_booleans */
bStopCM = flags & CGLO_STOPCM;
bGStat = flags & CGLO_GSTAT;
bReadEkin = (flags & CGLO_READEKIN);
bScaleEkin = (flags & CGLO_SCALEEKIN);
bEner = flags & CGLO_ENERGY;
bTemp = flags & CGLO_TEMPERATURE;
bPres = (flags & CGLO_PRESSURE);
bConstrain = (flags & CGLO_CONSTRAINT);
/* we calculate a full state kinetic energy either with full-step velocity verlet
or half step where we need the pressure */
bEkinAveVel = (ir->eI == eiVV || (ir->eI == eiVVAK && bPres) || bReadEkin);
/* in initalization, it sums the shake virial in vv, and to
sums ekinh_old in leapfrog (or if we are calculating ekinh_old) for other reasons */
/* ########## Kinetic energy ############## */
if (bTemp)
{
/* Non-equilibrium MD: this is parallellized, but only does communication
* when there really is NEMD.
*/
if (PAR(cr) && (ekind->bNEMD))
{
accumulate_u(cr, &(ir->opts), ekind);
}
debug_gmx();
if (bReadEkin)
{
restore_ekinstate_from_state(cr, ekind, &state_global->ekinstate);
}
else
{
calc_ke_part(state, &(ir->opts), mdatoms, ekind, nrnb, bEkinAveVel);
}
debug_gmx();
}
/* Calculate center of mass velocity if necessary, also parallellized */
if (bStopCM)
{
calc_vcm_grp(0, mdatoms->homenr, mdatoms,
state->x, state->v, vcm);
}
if (bTemp || bStopCM || bPres || bEner || bConstrain)
{
if (!bGStat)
{
/* We will not sum ekinh_old,
* so signal that we still have to do it.
*/
*bSumEkinhOld = TRUE;
}
else
{
if (gs != NULL)
{
for (i = 0; i < eglsNR; i++)
{
gs_buf[i] = gs->sig[i];
}
}
if (PAR(cr))
{
wallcycle_start(wcycle, ewcMoveE);
global_stat(fplog, gstat, cr, enerd, force_vir, shake_vir, mu_tot,
ir, ekind, constr, bStopCM ? vcm : NULL,
gs != NULL ? eglsNR : 0, gs_buf,
top_global, state,
*bSumEkinhOld, flags);
wallcycle_stop(wcycle, ewcMoveE);
}
if (gs != NULL)
{
if (MULTISIM(cr) && bInterSimGS)
{
if (MASTER(cr))
{
/* Communicate the signals between the simulations */
gmx_sum_sim(eglsNR, gs_buf, cr->ms);
}
/* Communicate the signals form the master to the others */
gmx_bcast(eglsNR*sizeof(gs_buf[0]), gs_buf, cr);
}
for (i = 0; i < eglsNR; i++)
{
if (bInterSimGS || gs_simlocal[i])
{
/* Set the communicated signal only when it is non-zero,
* since signals might not be processed at each MD step.
*/
gsi = (gs_buf[i] >= 0 ?
(int)(gs_buf[i] + 0.5) :
(int)(gs_buf[i] - 0.5));
if (gsi != 0)
{
gs->set[i] = gsi;
}
/* Turn off the local signal */
gs->sig[i] = 0;
}
}
}
*bSumEkinhOld = FALSE;
}
}
if (!ekind->bNEMD && debug && bTemp && (vcm->nr > 0))
{
correct_ekin(debug,
0, mdatoms->homenr,
state->v, vcm->group_p[0],
mdatoms->massT, mdatoms->tmass, ekind->ekin);
}
/* Do center of mass motion removal */
if (bStopCM)
{
check_cm_grp(fplog, vcm, ir, 1);
do_stopcm_grp(0, mdatoms->homenr, mdatoms->cVCM,
state->x, state->v, vcm);
inc_nrnb(nrnb, eNR_STOPCM, mdatoms->homenr);
}
if (bEner)
{
/* Calculate the amplitude of the cosine velocity profile */
ekind->cosacc.vcos = ekind->cosacc.mvcos/mdatoms->tmass;
}
if (bTemp)
{
/* Sum the kinetic energies of the groups & calc temp */
/* compute full step kinetic energies if vv, or if vv-avek and we are computing the pressure with IR_NPT_TROTTER */
/* three maincase: VV with AveVel (md-vv), vv with AveEkin (md-vv-avek), leap with AveEkin (md).
Leap with AveVel is not supported; it's not clear that it will actually work.
bEkinAveVel: If TRUE, we simply multiply ekin by ekinscale to get a full step kinetic energy.
If FALSE, we average ekinh_old and ekinh*ekinscale_nhc to get an averaged half step kinetic energy.
*/
enerd->term[F_TEMP] = sum_ekin(&(ir->opts), ekind, &dvdl_ekin,
bEkinAveVel, bScaleEkin);
enerd->dvdl_lin[efptMASS] = (double) dvdl_ekin;
enerd->term[F_EKIN] = trace(ekind->ekin);
}
/* ########## Long range energy information ###### */
if (bEner || bPres || bConstrain)
{
calc_dispcorr(ir, fr, top_global->natoms, box, state->lambda[efptVDW],
corr_pres, corr_vir, &prescorr, &enercorr, &dvdlcorr);
}
if (bEner)
{
enerd->term[F_DISPCORR] = enercorr;
enerd->term[F_EPOT] += enercorr;
enerd->term[F_DVDL_VDW] += dvdlcorr;
}
/* ########## Now pressure ############## */
if (bPres || bConstrain)
{
m_add(force_vir, shake_vir, total_vir);
/* Calculate pressure and apply LR correction if PPPM is used.
* Use the box from last timestep since we already called update().
*/
enerd->term[F_PRES] = calc_pres(fr->ePBC, ir->nwall, box, ekind->ekin, total_vir, pres);
/* Calculate long range corrections to pressure and energy */
/* this adds to enerd->term[F_PRES] and enerd->term[F_ETOT],
and computes enerd->term[F_DISPCORR]. Also modifies the
total_vir and pres tesors */
m_add(total_vir, corr_vir, total_vir);
m_add(pres, corr_pres, pres);
enerd->term[F_PDISPCORR] = prescorr;
enerd->term[F_PRES] += prescorr;
}
}
/*! \brief Support handling restarts
*
* \todo Clean this up (next patch)
*
* Read just the simulation 'generation' and with bTryToAppendFiles check files.
* This is is needed at the beginning of mdrun,
* to be able to rename the logfile correctly.
* When file appending is requested, checks which output files are present,
* and returns TRUE/FALSE in bDoAppendFiles if all or none are present.
* If only some output files are present, give a fatal error.
* When bDoAppendFiles is TRUE upon return, bAddPart will tell whether the simulation part
* needs to be added to the output file name.
*
* This routine cannot print tons of data, since it is called before
* the log file is opened. */
static void
read_checkpoint_data(const char *filename, int *simulation_part,
t_commrec *cr,
gmx_bool bTryToAppendFiles,
int nfile, const t_filenm fnm[],
const char *part_suffix,
gmx_bool *bAddPart,
gmx_bool *bDoAppendFiles)
{
t_fileio *fp;
int nfiles;
gmx_file_position_t *outputfiles;
int nexist, f;
char *fn, suf_up[STRLEN];
*bDoAppendFiles = FALSE;
if (SIMMASTER(cr))
{
if (!gmx_fexist(filename) || (!(fp = gmx_fio_open(filename, "r")) ))
{
*simulation_part = 0;
}
else
{
read_checkpoint_simulation_part_and_filenames(fp,
simulation_part,
&nfiles,
&outputfiles);
if (bTryToAppendFiles)
{
nexist = 0;
for (f = 0; f < nfiles; f++)
{
if (exist_output_file(outputfiles[f].filename, nfile, fnm))
{
nexist++;
}
}
if (nexist == nfiles)
{
*bDoAppendFiles = bTryToAppendFiles;
}
else if (nexist > 0)
{
fprintf(stderr,
"Output file appending has been requested,\n"
"but some output files listed in the checkpoint file %s\n"
"are not present or are named differently by the current program:\n",
filename);
fprintf(stderr, "output files present:");
for (f = 0; f < nfiles; f++)
{
if (exist_output_file(outputfiles[f].filename,
nfile, fnm))
{
fprintf(stderr, " %s", outputfiles[f].filename);
}
}
fprintf(stderr, "\n");
fprintf(stderr, "output files not present or named differently:");
for (f = 0; f < nfiles; f++)
{
if (!exist_output_file(outputfiles[f].filename,
nfile, fnm))
{
fprintf(stderr, " %s", outputfiles[f].filename);
}
}
fprintf(stderr, "\n");
gmx_fatal(FARGS, "File appending requested, but %d of the %d output files are not present or are named differently", nfiles-nexist, nfiles);
}
}
if (*bDoAppendFiles)
{
if (nfiles == 0)
{
gmx_fatal(FARGS, "File appending requested, but no output file information is stored in the checkpoint file");
}
fn = outputfiles[0].filename;
if (strlen(fn) < 4 ||
gmx_strcasecmp(fn+strlen(fn)-4, ftp2ext(efLOG)) == 0)
{
gmx_fatal(FARGS, "File appending requested, but the log file is not the first file listed in the checkpoint file");
}
/* Set bAddPart to whether the suffix string '.part' is present
* in the log file name.
*/
strcpy(suf_up, part_suffix);
upstring(suf_up);
*bAddPart = (strstr(fn, part_suffix) != NULL ||
strstr(fn, suf_up) != NULL);
}
sfree(outputfiles);
}
}
if (PAR(cr))
{
gmx_bcast(sizeof(*simulation_part), simulation_part, cr);
if (*simulation_part > 0 && bTryToAppendFiles)
{
gmx_bcast(sizeof(*bDoAppendFiles), bDoAppendFiles, cr);
gmx_bcast(sizeof(*bAddPart), bAddPart, cr);
}
}
}
real dd_choose_grid(FILE *fplog,
t_commrec *cr, gmx_domdec_t *dd, t_inputrec *ir,
gmx_mtop_t *mtop, matrix box, gmx_ddbox_t *ddbox,
gmx_bool bDynLoadBal, real dlb_scale,
real cellsize_limit, real cutoff_dd,
gmx_bool bInterCGBondeds)
{
gmx_int64_t nnodes_div, ldiv;
real limit;
if (MASTER(cr))
{
nnodes_div = cr->nnodes;
if (EEL_PME(ir->coulombtype))
{
if (cr->npmenodes > 0)
{
if (cr->npmenodes >= cr->nnodes)
{
gmx_fatal(FARGS,
"Cannot have %d separate PME ranks with just %d total ranks",
cr->npmenodes, cr->nnodes);
}
/* If the user purposely selected the number of PME nodes,
* only check for large primes in the PP node count.
*/
nnodes_div -= cr->npmenodes;
}
}
else
{
cr->npmenodes = 0;
}
if (nnodes_div > 12)
{
ldiv = largest_divisor(nnodes_div);
/* Check if the largest divisor is more than nnodes^2/3 */
if (ldiv*ldiv*ldiv > nnodes_div*nnodes_div)
{
gmx_fatal(FARGS, "The number of ranks you selected (%d) contains a large prime factor %d. In most cases this will lead to bad performance. Choose a number with smaller prime factors or set the decomposition (option -dd) manually.",
nnodes_div, ldiv);
}
}
if (EEL_PME(ir->coulombtype))
{
if (cr->npmenodes < 0)
{
/* Use PME nodes when the number of nodes is more than 16 */
if (cr->nnodes <= 18)
{
cr->npmenodes = 0;
if (fplog)
{
fprintf(fplog, "Using %d separate PME ranks, as there are too few total\n ranks for efficient splitting\n", cr->npmenodes);
}
}
else
{
cr->npmenodes = guess_npme(fplog, mtop, ir, box, cr->nnodes);
if (fplog)
{
fprintf(fplog, "Using %d separate PME ranks, as guessed by mdrun\n", cr->npmenodes);
}
}
}
else
{
if (fplog)
{
fprintf(fplog, "Using %d separate PME ranks, per user request\n", cr->npmenodes);
}
}
}
limit = optimize_ncells(fplog, cr->nnodes, cr->npmenodes,
bDynLoadBal, dlb_scale,
mtop, box, ddbox, ir, dd,
cellsize_limit, cutoff_dd,
bInterCGBondeds,
dd->nc);
}
else
{
limit = 0;
}
/* Communicate the information set by the master to all nodes */
gmx_bcast(sizeof(dd->nc), dd->nc, cr);
if (EEL_PME(ir->coulombtype))
{
gmx_bcast(sizeof(ir->nkx), &ir->nkx, cr);
gmx_bcast(sizeof(ir->nky), &ir->nky, cr);
gmx_bcast(sizeof(cr->npmenodes), &cr->npmenodes, cr);
}
else
{
cr->npmenodes = 0;
}
return limit;
}
void gmx_log_open(const char *lognm, const t_commrec *cr, gmx_bool bMasterOnly,
gmx_bool bAppendFiles, FILE** fplog)
{
int len, pid;
char buf[256], host[256];
time_t t;
char timebuf[STRLEN];
FILE *fp = *fplog;
char *tmpnm;
debug_gmx();
/* Communicate the filename for logfile */
if (cr->nnodes > 1 && !bMasterOnly
#ifdef GMX_THREAD_MPI
/* With thread MPI the non-master log files are opened later
* when the files names are already known on all nodes.
*/
&& FALSE
#endif
)
{
if (MASTER(cr))
{
len = strlen(lognm) + 1;
}
gmx_bcast(sizeof(len), &len, cr);
if (!MASTER(cr))
{
snew(tmpnm, len+8);
}
else
{
tmpnm = gmx_strdup(lognm);
}
gmx_bcast(len*sizeof(*tmpnm), tmpnm, cr);
}
else
{
tmpnm = gmx_strdup(lognm);
}
debug_gmx();
if (!bMasterOnly && !MASTER(cr))
{
/* Since log always ends with '.log' let's use this info */
par_fn(tmpnm, efLOG, cr, FALSE, !bMasterOnly, buf, 255);
fp = gmx_fio_fopen(buf, bAppendFiles ? "a+" : "w+" );
}
else if (!bAppendFiles)
{
fp = gmx_fio_fopen(tmpnm, bAppendFiles ? "a+" : "w+" );
}
sfree(tmpnm);
gmx_fatal_set_log_file(fp);
/* Get some machine parameters */
gmx_gethostname(host, 256);
time(&t);
#ifndef NO_GETPID
# ifdef GMX_NATIVE_WINDOWS
pid = _getpid();
# else
pid = getpid();
# endif
#else
pid = 0;
#endif
if (bAppendFiles)
{
fprintf(fp,
"\n"
"\n"
"-----------------------------------------------------------\n"
"Restarting from checkpoint, appending to previous log file.\n"
"\n"
);
}
gmx_ctime_r(&t, timebuf, STRLEN);
fprintf(fp,
"Log file opened on %s"
"Host: %s pid: %d rank ID: %d number of ranks: %d\n",
timebuf, host, pid, cr->nodeid, cr->nnodes);
try
{
gmx::BinaryInformationSettings settings;
settings.extendedInfo(true);
settings.copyright(!bAppendFiles);
gmx::printBinaryInformation(fp, gmx::getProgramContext(), settings);
}
GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
fprintf(fp, "\n\n");
fflush(fp);
debug_gmx();
*fplog = fp;
}
int mdrunner(gmx_hw_opt_t *hw_opt,
FILE *fplog, t_commrec *cr, int nfile,
const t_filenm fnm[], const output_env_t oenv, gmx_bool bVerbose,
gmx_bool bCompact, int nstglobalcomm,
ivec ddxyz, int dd_node_order, real rdd, real rconstr,
const char *dddlb_opt, real dlb_scale,
const char *ddcsx, const char *ddcsy, const char *ddcsz,
const char *nbpu_opt, int nstlist_cmdline,
gmx_int64_t nsteps_cmdline, int nstepout, int resetstep,
int gmx_unused nmultisim, int repl_ex_nst, int repl_ex_nex,
int repl_ex_seed, real pforce, real cpt_period, real max_hours,
int imdport, unsigned long Flags)
{
gmx_bool bForceUseGPU, bTryUseGPU, bRerunMD;
t_inputrec *inputrec;
t_state *state = NULL;
matrix box;
gmx_ddbox_t ddbox = {0};
int npme_major, npme_minor;
t_nrnb *nrnb;
gmx_mtop_t *mtop = NULL;
t_mdatoms *mdatoms = NULL;
t_forcerec *fr = NULL;
t_fcdata *fcd = NULL;
real ewaldcoeff_q = 0;
real ewaldcoeff_lj = 0;
struct gmx_pme_t **pmedata = NULL;
gmx_vsite_t *vsite = NULL;
gmx_constr_t constr;
int nChargePerturbed = -1, nTypePerturbed = 0, status;
gmx_wallcycle_t wcycle;
gmx_bool bReadEkin;
gmx_walltime_accounting_t walltime_accounting = NULL;
int rc;
gmx_int64_t reset_counters;
gmx_edsam_t ed = NULL;
int nthreads_pme = 1;
int nthreads_pp = 1;
gmx_membed_t membed = NULL;
gmx_hw_info_t *hwinfo = NULL;
/* The master rank decides early on bUseGPU and broadcasts this later */
gmx_bool bUseGPU = FALSE;
/* CAUTION: threads may be started later on in this function, so
cr doesn't reflect the final parallel state right now */
snew(inputrec, 1);
snew(mtop, 1);
if (Flags & MD_APPENDFILES)
{
fplog = NULL;
}
bRerunMD = (Flags & MD_RERUN);
bForceUseGPU = (strncmp(nbpu_opt, "gpu", 3) == 0);
bTryUseGPU = (strncmp(nbpu_opt, "auto", 4) == 0) || bForceUseGPU;
/* Detect hardware, gather information. This is an operation that is
* global for this process (MPI rank). */
hwinfo = gmx_detect_hardware(fplog, cr, bTryUseGPU);
gmx_print_detected_hardware(fplog, cr, hwinfo);
if (fplog != NULL)
{
/* Print references after all software/hardware printing */
please_cite(fplog, "Abraham2015");
please_cite(fplog, "Pall2015");
please_cite(fplog, "Pronk2013");
please_cite(fplog, "Hess2008b");
please_cite(fplog, "Spoel2005a");
please_cite(fplog, "Lindahl2001a");
please_cite(fplog, "Berendsen95a");
}
snew(state, 1);
if (SIMMASTER(cr))
{
/* Read (nearly) all data required for the simulation */
read_tpx_state(ftp2fn(efTPR, nfile, fnm), inputrec, state, NULL, mtop);
if (inputrec->cutoff_scheme == ecutsVERLET)
{
/* Here the master rank decides if all ranks will use GPUs */
bUseGPU = (hwinfo->gpu_info.n_dev_compatible > 0 ||
getenv("GMX_EMULATE_GPU") != NULL);
/* TODO add GPU kernels for this and replace this check by:
* (bUseGPU && (ir->vdwtype == evdwPME &&
* ir->ljpme_combination_rule == eljpmeLB))
* update the message text and the content of nbnxn_acceleration_supported.
*/
if (bUseGPU &&
!nbnxn_gpu_acceleration_supported(fplog, cr, inputrec, bRerunMD))
{
/* Fallback message printed by nbnxn_acceleration_supported */
if (bForceUseGPU)
{
gmx_fatal(FARGS, "GPU acceleration requested, but not supported with the given input settings");
}
bUseGPU = FALSE;
}
prepare_verlet_scheme(fplog, cr,
inputrec, nstlist_cmdline, mtop, state->box,
bUseGPU);
}
else
{
if (nstlist_cmdline > 0)
{
gmx_fatal(FARGS, "Can not set nstlist with the group cut-off scheme");
}
if (hwinfo->gpu_info.n_dev_compatible > 0)
{
md_print_warn(cr, fplog,
"NOTE: GPU(s) found, but the current simulation can not use GPUs\n"
" To use a GPU, set the mdp option: cutoff-scheme = Verlet\n");
}
if (bForceUseGPU)
{
gmx_fatal(FARGS, "GPU requested, but can't be used without cutoff-scheme=Verlet");
}
#ifdef GMX_TARGET_BGQ
md_print_warn(cr, fplog,
"NOTE: There is no SIMD implementation of the group scheme kernels on\n"
" BlueGene/Q. You will observe better performance from using the\n"
" Verlet cut-off scheme.\n");
#endif
}
if (inputrec->eI == eiSD2)
{
md_print_warn(cr, fplog, "The stochastic dynamics integrator %s is deprecated, since\n"
"it is slower than integrator %s and is slightly less accurate\n"
"with constraints. Use the %s integrator.",
ei_names[inputrec->eI], ei_names[eiSD1], ei_names[eiSD1]);
}
}
/* Check and update the hardware options for internal consistency */
check_and_update_hw_opt_1(hw_opt, cr);
/* Early check for externally set process affinity. */
gmx_check_thread_affinity_set(fplog, cr,
hw_opt, hwinfo->nthreads_hw_avail, FALSE);
#ifdef GMX_THREAD_MPI
if (SIMMASTER(cr))
{
if (cr->npmenodes > 0 && hw_opt->nthreads_tmpi <= 0)
{
gmx_fatal(FARGS, "You need to explicitly specify the number of MPI threads (-ntmpi) when using separate PME ranks");
}
/* Since the master knows the cut-off scheme, update hw_opt for this.
* This is done later for normal MPI and also once more with tMPI
* for all tMPI ranks.
*/
check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme);
/* NOW the threads will be started: */
hw_opt->nthreads_tmpi = get_nthreads_mpi(hwinfo,
hw_opt,
inputrec, mtop,
cr, fplog, bUseGPU);
if (hw_opt->nthreads_tmpi > 1)
{
t_commrec *cr_old = cr;
/* now start the threads. */
cr = mdrunner_start_threads(hw_opt, fplog, cr_old, nfile, fnm,
oenv, bVerbose, bCompact, nstglobalcomm,
ddxyz, dd_node_order, rdd, rconstr,
dddlb_opt, dlb_scale, ddcsx, ddcsy, ddcsz,
nbpu_opt, nstlist_cmdline,
nsteps_cmdline, nstepout, resetstep, nmultisim,
repl_ex_nst, repl_ex_nex, repl_ex_seed, pforce,
cpt_period, max_hours,
Flags);
/* the main thread continues here with a new cr. We don't deallocate
the old cr because other threads may still be reading it. */
if (cr == NULL)
{
gmx_comm("Failed to spawn threads");
}
}
}
#endif
/* END OF CAUTION: cr is now reliable */
/* g_membed initialisation *
* Because we change the mtop, init_membed is called before the init_parallel *
* (in case we ever want to make it run in parallel) */
if (opt2bSet("-membed", nfile, fnm))
{
if (MASTER(cr))
{
fprintf(stderr, "Initializing membed");
}
membed = init_membed(fplog, nfile, fnm, mtop, inputrec, state, cr, &cpt_period);
}
if (PAR(cr))
{
/* now broadcast everything to the non-master nodes/threads: */
init_parallel(cr, inputrec, mtop);
/* The master rank decided on the use of GPUs,
* broadcast this information to all ranks.
*/
gmx_bcast_sim(sizeof(bUseGPU), &bUseGPU, cr);
}
if (fplog != NULL)
{
pr_inputrec(fplog, 0, "Input Parameters", inputrec, FALSE);
fprintf(fplog, "\n");
}
/* now make sure the state is initialized and propagated */
set_state_entries(state, inputrec);
/* A parallel command line option consistency check that we can
only do after any threads have started. */
if (!PAR(cr) &&
(ddxyz[XX] > 1 || ddxyz[YY] > 1 || ddxyz[ZZ] > 1 || cr->npmenodes > 0))
{
gmx_fatal(FARGS,
"The -dd or -npme option request a parallel simulation, "
#ifndef GMX_MPI
"but %s was compiled without threads or MPI enabled"
#else
#ifdef GMX_THREAD_MPI
"but the number of threads (option -nt) is 1"
#else
"but %s was not started through mpirun/mpiexec or only one rank was requested through mpirun/mpiexec"
#endif
#endif
, output_env_get_program_display_name(oenv)
);
}
if (bRerunMD &&
(EI_ENERGY_MINIMIZATION(inputrec->eI) || eiNM == inputrec->eI))
{
gmx_fatal(FARGS, "The .mdp file specified an energy mininization or normal mode algorithm, and these are not compatible with mdrun -rerun");
}
if (can_use_allvsall(inputrec, TRUE, cr, fplog) && DOMAINDECOMP(cr))
{
gmx_fatal(FARGS, "All-vs-all loops do not work with domain decomposition, use a single MPI rank");
}
if (!(EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype)))
{
if (cr->npmenodes > 0)
{
gmx_fatal_collective(FARGS, cr, NULL,
"PME-only ranks are requested, but the system does not use PME for electrostatics or LJ");
}
cr->npmenodes = 0;
}
if (bUseGPU && cr->npmenodes < 0)
{
/* With GPUs we don't automatically use PME-only ranks. PME ranks can
* improve performance with many threads per GPU, since our OpenMP
* scaling is bad, but it's difficult to automate the setup.
*/
cr->npmenodes = 0;
}
#ifdef GMX_FAHCORE
if (MASTER(cr))
{
fcRegisterSteps(inputrec->nsteps, inputrec->init_step);
}
#endif
/* NMR restraints must be initialized before load_checkpoint,
* since with time averaging the history is added to t_state.
* For proper consistency check we therefore need to extend
* t_state here.
* So the PME-only nodes (if present) will also initialize
* the distance restraints.
*/
snew(fcd, 1);
/* This needs to be called before read_checkpoint to extend the state */
init_disres(fplog, mtop, inputrec, cr, fcd, state, repl_ex_nst > 0);
init_orires(fplog, mtop, state->x, inputrec, cr, &(fcd->orires),
state);
if (DEFORM(*inputrec))
{
/* Store the deform reference box before reading the checkpoint */
if (SIMMASTER(cr))
{
copy_mat(state->box, box);
}
if (PAR(cr))
{
gmx_bcast(sizeof(box), box, cr);
}
/* Because we do not have the update struct available yet
* in which the reference values should be stored,
* we store them temporarily in static variables.
* This should be thread safe, since they are only written once
* and with identical values.
*/
tMPI_Thread_mutex_lock(&deform_init_box_mutex);
deform_init_init_step_tpx = inputrec->init_step;
copy_mat(box, deform_init_box_tpx);
tMPI_Thread_mutex_unlock(&deform_init_box_mutex);
}
if (opt2bSet("-cpi", nfile, fnm))
{
/* Check if checkpoint file exists before doing continuation.
* This way we can use identical input options for the first and subsequent runs...
*/
if (gmx_fexist_master(opt2fn_master("-cpi", nfile, fnm, cr), cr) )
{
load_checkpoint(opt2fn_master("-cpi", nfile, fnm, cr), &fplog,
cr, ddxyz,
inputrec, state, &bReadEkin,
(Flags & MD_APPENDFILES),
(Flags & MD_APPENDFILESSET));
if (bReadEkin)
{
Flags |= MD_READ_EKIN;
}
}
}
if (MASTER(cr) && (Flags & MD_APPENDFILES))
{
gmx_log_open(ftp2fn(efLOG, nfile, fnm), cr,
Flags, &fplog);
}
/* override nsteps with value from cmdline */
override_nsteps_cmdline(fplog, nsteps_cmdline, inputrec, cr);
if (SIMMASTER(cr))
{
copy_mat(state->box, box);
}
if (PAR(cr))
{
gmx_bcast(sizeof(box), box, cr);
}
/* Essential dynamics */
if (opt2bSet("-ei", nfile, fnm))
{
/* Open input and output files, allocate space for ED data structure */
ed = ed_open(mtop->natoms, &state->edsamstate, nfile, fnm, Flags, oenv, cr);
}
if (PAR(cr) && !(EI_TPI(inputrec->eI) ||
inputrec->eI == eiNM))
{
cr->dd = init_domain_decomposition(fplog, cr, Flags, ddxyz, rdd, rconstr,
dddlb_opt, dlb_scale,
ddcsx, ddcsy, ddcsz,
mtop, inputrec,
box, state->x,
&ddbox, &npme_major, &npme_minor);
make_dd_communicators(fplog, cr, dd_node_order);
/* Set overallocation to avoid frequent reallocation of arrays */
set_over_alloc_dd(TRUE);
}
else
{
/* PME, if used, is done on all nodes with 1D decomposition */
cr->npmenodes = 0;
cr->duty = (DUTY_PP | DUTY_PME);
npme_major = 1;
npme_minor = 1;
if (inputrec->ePBC == epbcSCREW)
{
gmx_fatal(FARGS,
"pbc=%s is only implemented with domain decomposition",
epbc_names[inputrec->ePBC]);
}
}
if (PAR(cr))
{
/* After possible communicator splitting in make_dd_communicators.
* we can set up the intra/inter node communication.
*/
gmx_setup_nodecomm(fplog, cr);
}
/* Initialize per-physical-node MPI process/thread ID and counters. */
gmx_init_intranode_counters(cr);
#ifdef GMX_MPI
if (MULTISIM(cr))
{
md_print_info(cr, fplog,
"This is simulation %d out of %d running as a composite GROMACS\n"
"multi-simulation job. Setup for this simulation:\n\n",
cr->ms->sim, cr->ms->nsim);
}
md_print_info(cr, fplog, "Using %d MPI %s\n",
cr->nnodes,
#ifdef GMX_THREAD_MPI
cr->nnodes == 1 ? "thread" : "threads"
#else
cr->nnodes == 1 ? "process" : "processes"
#endif
);
fflush(stderr);
#endif
/* Check and update hw_opt for the cut-off scheme */
check_and_update_hw_opt_2(hw_opt, inputrec->cutoff_scheme);
/* Check and update hw_opt for the number of MPI ranks */
check_and_update_hw_opt_3(hw_opt);
gmx_omp_nthreads_init(fplog, cr,
hwinfo->nthreads_hw_avail,
hw_opt->nthreads_omp,
hw_opt->nthreads_omp_pme,
(cr->duty & DUTY_PP) == 0,
inputrec->cutoff_scheme == ecutsVERLET);
#ifndef NDEBUG
if (integrator[inputrec->eI].func != do_tpi &&
inputrec->cutoff_scheme == ecutsVERLET)
{
gmx_feenableexcept();
}
#endif
if (bUseGPU)
{
/* Select GPU id's to use */
gmx_select_gpu_ids(fplog, cr, &hwinfo->gpu_info, bForceUseGPU,
&hw_opt->gpu_opt);
}
else
{
/* Ignore (potentially) manually selected GPUs */
hw_opt->gpu_opt.n_dev_use = 0;
}
/* check consistency across ranks of things like SIMD
* support and number of GPUs selected */
gmx_check_hw_runconf_consistency(fplog, hwinfo, cr, hw_opt, bUseGPU);
/* Now that we know the setup is consistent, check for efficiency */
check_resource_division_efficiency(hwinfo, hw_opt, Flags & MD_NTOMPSET,
cr, fplog);
if (DOMAINDECOMP(cr))
{
/* When we share GPUs over ranks, we need to know this for the DLB */
dd_setup_dlb_resource_sharing(cr, hwinfo, hw_opt);
}
/* getting number of PP/PME threads
PME: env variable should be read only on one node to make sure it is
identical everywhere;
*/
/* TODO nthreads_pp is only used for pinning threads.
* This is a temporary solution until we have a hw topology library.
*/
nthreads_pp = gmx_omp_nthreads_get(emntNonbonded);
nthreads_pme = gmx_omp_nthreads_get(emntPME);
wcycle = wallcycle_init(fplog, resetstep, cr, nthreads_pp, nthreads_pme);
if (PAR(cr))
{
/* Master synchronizes its value of reset_counters with all nodes
* including PME only nodes */
reset_counters = wcycle_get_reset_counters(wcycle);
gmx_bcast_sim(sizeof(reset_counters), &reset_counters, cr);
wcycle_set_reset_counters(wcycle, reset_counters);
}
snew(nrnb, 1);
if (cr->duty & DUTY_PP)
{
bcast_state(cr, state);
/* Initiate forcerecord */
fr = mk_forcerec();
fr->hwinfo = hwinfo;
fr->gpu_opt = &hw_opt->gpu_opt;
init_forcerec(fplog, oenv, fr, fcd, inputrec, mtop, cr, box,
opt2fn("-table", nfile, fnm),
opt2fn("-tabletf", nfile, fnm),
opt2fn("-tablep", nfile, fnm),
opt2fn("-tableb", nfile, fnm),
nbpu_opt,
FALSE,
pforce);
/* version for PCA_NOT_READ_NODE (see md.c) */
/*init_forcerec(fplog,fr,fcd,inputrec,mtop,cr,box,FALSE,
"nofile","nofile","nofile","nofile",FALSE,pforce);
*/
/* Initialize QM-MM */
if (fr->bQMMM)
{
init_QMMMrec(cr, mtop, inputrec, fr);
}
/* Initialize the mdatoms structure.
* mdatoms is not filled with atom data,
* as this can not be done now with domain decomposition.
*/
mdatoms = init_mdatoms(fplog, mtop, inputrec->efep != efepNO);
/* Initialize the virtual site communication */
vsite = init_vsite(mtop, cr, FALSE);
calc_shifts(box, fr->shift_vec);
/* With periodic molecules the charge groups should be whole at start up
* and the virtual sites should not be far from their proper positions.
*/
if (!inputrec->bContinuation && MASTER(cr) &&
!(inputrec->ePBC != epbcNONE && inputrec->bPeriodicMols))
{
/* Make molecules whole at start of run */
if (fr->ePBC != epbcNONE)
{
do_pbc_first_mtop(fplog, inputrec->ePBC, box, mtop, state->x);
}
if (vsite)
{
/* Correct initial vsite positions are required
* for the initial distribution in the domain decomposition
* and for the initial shell prediction.
*/
construct_vsites_mtop(vsite, mtop, state->x);
}
}
if (EEL_PME(fr->eeltype) || EVDW_PME(fr->vdwtype))
{
ewaldcoeff_q = fr->ewaldcoeff_q;
ewaldcoeff_lj = fr->ewaldcoeff_lj;
pmedata = &fr->pmedata;
}
else
{
pmedata = NULL;
}
}
else
{
/* This is a PME only node */
/* We don't need the state */
done_state(state);
ewaldcoeff_q = calc_ewaldcoeff_q(inputrec->rcoulomb, inputrec->ewald_rtol);
ewaldcoeff_lj = calc_ewaldcoeff_lj(inputrec->rvdw, inputrec->ewald_rtol_lj);
snew(pmedata, 1);
}
if (hw_opt->thread_affinity != threadaffOFF)
{
/* Before setting affinity, check whether the affinity has changed
* - which indicates that probably the OpenMP library has changed it
* since we first checked).
*/
gmx_check_thread_affinity_set(fplog, cr,
hw_opt, hwinfo->nthreads_hw_avail, TRUE);
/* Set the CPU affinity */
gmx_set_thread_affinity(fplog, cr, hw_opt, hwinfo);
}
/* Initiate PME if necessary,
* either on all nodes or on dedicated PME nodes only. */
if (EEL_PME(inputrec->coulombtype) || EVDW_PME(inputrec->vdwtype))
{
if (mdatoms)
{
nChargePerturbed = mdatoms->nChargePerturbed;
if (EVDW_PME(inputrec->vdwtype))
{
nTypePerturbed = mdatoms->nTypePerturbed;
}
}
if (cr->npmenodes > 0)
{
/* The PME only nodes need to know nChargePerturbed(FEP on Q) and nTypePerturbed(FEP on LJ)*/
gmx_bcast_sim(sizeof(nChargePerturbed), &nChargePerturbed, cr);
gmx_bcast_sim(sizeof(nTypePerturbed), &nTypePerturbed, cr);
}
if (cr->duty & DUTY_PME)
{
status = gmx_pme_init(pmedata, cr, npme_major, npme_minor, inputrec,
mtop ? mtop->natoms : 0, nChargePerturbed, nTypePerturbed,
(Flags & MD_REPRODUCIBLE), nthreads_pme);
if (status != 0)
{
gmx_fatal(FARGS, "Error %d initializing PME", status);
}
}
}
if (integrator[inputrec->eI].func == do_md)
{
/* Turn on signal handling on all nodes */
/*
* (A user signal from the PME nodes (if any)
* is communicated to the PP nodes.
*/
signal_handler_install();
}
if (cr->duty & DUTY_PP)
{
/* Assumes uniform use of the number of OpenMP threads */
walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntDefault));
if (inputrec->bPull)
{
/* Initialize pull code */
inputrec->pull_work =
init_pull(fplog, inputrec->pull, inputrec, nfile, fnm,
mtop, cr, oenv, inputrec->fepvals->init_lambda,
EI_DYNAMICS(inputrec->eI) && MASTER(cr), Flags);
}
if (inputrec->bRot)
{
/* Initialize enforced rotation code */
init_rot(fplog, inputrec, nfile, fnm, cr, state->x, box, mtop, oenv,
bVerbose, Flags);
}
if (inputrec->eSwapCoords != eswapNO)
{
/* Initialize ion swapping code */
init_swapcoords(fplog, bVerbose, inputrec, opt2fn_master("-swap", nfile, fnm, cr),
mtop, state->x, state->box, &state->swapstate, cr, oenv, Flags);
}
constr = init_constraints(fplog, mtop, inputrec, ed, state, cr);
if (DOMAINDECOMP(cr))
{
GMX_RELEASE_ASSERT(fr, "fr was NULL while cr->duty was DUTY_PP");
dd_init_bondeds(fplog, cr->dd, mtop, vsite, inputrec,
Flags & MD_DDBONDCHECK, fr->cginfo_mb);
set_dd_parameters(fplog, cr->dd, dlb_scale, inputrec, &ddbox);
setup_dd_grid(fplog, cr->dd);
}
/* Now do whatever the user wants us to do (how flexible...) */
integrator[inputrec->eI].func(fplog, cr, nfile, fnm,
oenv, bVerbose, bCompact,
nstglobalcomm,
vsite, constr,
nstepout, inputrec, mtop,
fcd, state,
mdatoms, nrnb, wcycle, ed, fr,
repl_ex_nst, repl_ex_nex, repl_ex_seed,
membed,
cpt_period, max_hours,
imdport,
Flags,
walltime_accounting);
if (inputrec->bPull)
{
finish_pull(inputrec->pull_work);
}
if (inputrec->bRot)
{
finish_rot(inputrec->rot);
}
}
else
{
GMX_RELEASE_ASSERT(pmedata, "pmedata was NULL while cr->duty was not DUTY_PP");
/* do PME only */
walltime_accounting = walltime_accounting_init(gmx_omp_nthreads_get(emntPME));
gmx_pmeonly(*pmedata, cr, nrnb, wcycle, walltime_accounting, ewaldcoeff_q, ewaldcoeff_lj, inputrec);
}
wallcycle_stop(wcycle, ewcRUN);
/* Finish up, write some stuff
* if rerunMD, don't write last frame again
*/
finish_run(fplog, cr,
inputrec, nrnb, wcycle, walltime_accounting,
fr ? fr->nbv : NULL,
EI_DYNAMICS(inputrec->eI) && !MULTISIM(cr));
/* Free GPU memory and context */
free_gpu_resources(fr, cr, &hwinfo->gpu_info, fr ? fr->gpu_opt : NULL);
if (opt2bSet("-membed", nfile, fnm))
{
sfree(membed);
}
gmx_hardware_info_free(hwinfo);
/* Does what it says */
print_date_and_time(fplog, cr->nodeid, "Finished mdrun", gmx_gettime());
walltime_accounting_destroy(walltime_accounting);
/* PLUMED */
if(plumedswitch){
plumed_finalize(plumedmain);
}
/* END PLUMED */
/* Close logfile already here if we were appending to it */
if (MASTER(cr) && (Flags & MD_APPENDFILES))
{
gmx_log_close(fplog);
}
rc = (int)gmx_get_stop_condition();
done_ed(&ed);
#ifdef GMX_THREAD_MPI
/* we need to join all threads. The sub-threads join when they
exit this function, but the master thread needs to be told to
wait for that. */
if (PAR(cr) && MASTER(cr))
{
tMPI_Finalize();
}
#endif
return rc;
}
FILE *gmx_log_open(char *lognm,const t_commrec *cr,bool bMasterOnly, unsigned long Flags)
{
int len,testlen,pid;
char buf[256],host[256];
time_t t;
FILE *fp;
bool bAppend = Flags & MD_APPENDFILES;
debug_gmx();
/* Communicate the filename for logfile */
if (cr->nnodes > 1 && !bMasterOnly) {
if (MASTER(cr))
len = strlen(lognm)+1;
gmx_bcast(sizeof(len),&len,cr);
if (!MASTER(cr))
snew(lognm,len+8);
gmx_bcast(len*sizeof(*lognm),lognm,cr);
}
debug_gmx();
if (PAR(cr) && !bMasterOnly) {
/* Since log always ends with '.log' let's use this info */
par_fn(lognm,efLOG,cr,cr->ms!=NULL,buf,255);
fp = gmx_fio_fopen(buf, bAppend ? "a" : "w" );
} else {
fp = gmx_fio_fopen(lognm, bAppend ? "a" : "w" );
}
gmx_fatal_set_log_file(fp);
/* Get some machine parameters */
#ifdef HAVE_UNISTD_H
if( gethostname(host,255) != 0)
sprintf(host,"unknown");
#else
sprintf(host,"unknown");
#endif
time(&t);
#ifndef NO_GETPID
# if ((defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64) && !defined __CYGWIN__ && !defined __CYGWIN32__)
pid = _getpid();
# else
pid = getpid();
# endif
#else
pid = 0;
#endif
if(bAppend)
{
fprintf(fp,
"\n\n"
"-----------------------------------------------------------\n"
"Restarting from checkpoint, appending to previous log file.\n\n"
);
}
fprintf(fp,
"Log file opened on %s"
"Host: %s pid: %d nodeid: %d nnodes: %d\n",
ctime(&t),host,pid,cr->nodeid,cr->nnodes);
#if (defined BUILD_MACHINE && defined BUILD_TIME && defined BUILD_USER)
fprintf(fp,
"The Gromacs distribution was built %s by\n"
"%s (%s)\n\n\n",BUILD_TIME,BUILD_USER,BUILD_MACHINE);
#endif
fflush(fp);
debug_gmx();
return fp;
}
real dd_choose_grid(FILE *fplog,
t_commrec *cr,gmx_domdec_t *dd,t_inputrec *ir,
gmx_mtop_t *mtop,matrix box,gmx_ddbox_t *ddbox,
bool bDynLoadBal,real dlb_scale,
real cellsize_limit,real cutoff_dd,
bool bInterCGBondeds,bool bInterCGMultiBody)
{
int npme,nkx,nky;
real limit;
if (MASTER(cr))
{
if (EEL_PME(ir->coulombtype))
{
if (cr->npmenodes >= 0)
{
if (cr->nnodes <= 2 && cr->npmenodes > 0)
{
gmx_fatal(FARGS,
"Can not have separate PME nodes with 2 or less nodes");
}
}
else
{
if (cr->nnodes < 12 &&
pme_inconvenient_nnodes(ir->nkx,ir->nky,cr->nnodes) == 0)
{
cr->npmenodes = 0;
}
else
{
cr->npmenodes = guess_npme(fplog,mtop,ir,box,cr->nnodes);
}
}
if (fplog)
{
fprintf(fplog,"Using %d separate PME nodes\n",cr->npmenodes);
}
}
else
{
if (cr->npmenodes < 0)
{
cr->npmenodes = 0;
}
}
limit = optimize_ncells(fplog,cr->nnodes,cr->npmenodes,
bDynLoadBal,dlb_scale,
mtop,box,ddbox,ir,dd,
cellsize_limit,cutoff_dd,
bInterCGBondeds,bInterCGMultiBody,
dd->nc);
}
else
{
limit = 0;
}
/* Communicate the information set by the master to all nodes */
gmx_bcast(sizeof(dd->nc),dd->nc,cr);
if (EEL_PME(ir->coulombtype))
{
gmx_bcast(sizeof(ir->nkx),&ir->nkx,cr);
gmx_bcast(sizeof(ir->nky),&ir->nky,cr);
gmx_bcast(sizeof(cr->npmenodes),&cr->npmenodes,cr);
}
else
{
cr->npmenodes = 0;
}
return limit;
}
real dd_choose_grid(FILE *fplog,
t_commrec *cr,gmx_domdec_t *dd,t_inputrec *ir,
gmx_mtop_t *mtop,matrix box,gmx_ddbox_t *ddbox,
gmx_bool bDynLoadBal,real dlb_scale,
real cellsize_limit,real cutoff_dd,
gmx_bool bInterCGBondeds,gmx_bool bInterCGMultiBody)
{
int npme,nkx,nky;
int ldiv;
real limit;
if (MASTER(cr))
{
if (cr->nnodes > 12)
{
ldiv = largest_divisor(cr->nnodes);
/* Check if the largest divisor is more than nnodes^2/3 */
if (ldiv*ldiv*ldiv > cr->nnodes*cr->nnodes)
{
gmx_fatal(FARGS,"The number of nodes you selected (%d) contains a large prime factor %d. In most cases this will lead to bad performance. Choose a number with smaller prime factors or set the decomposition (option -dd) manually.",
cr->nnodes,ldiv);
}
}
if (EEL_PME(ir->coulombtype))
{
if (cr->npmenodes >= 0)
{
if (cr->nnodes <= 2 && cr->npmenodes > 0)
{
gmx_fatal(FARGS,
"Can not have separate PME nodes with 2 or less nodes");
}
}
else
{
if (cr->nnodes <= 10)
{
cr->npmenodes = 0;
}
else
{
cr->npmenodes = guess_npme(fplog,mtop,ir,box,cr->nnodes);
}
}
if (fplog)
{
fprintf(fplog,"Using %d separate PME nodes\n",cr->npmenodes);
}
}
else
{
if (cr->npmenodes < 0)
{
cr->npmenodes = 0;
}
}
limit = optimize_ncells(fplog,cr->nnodes,cr->npmenodes,
bDynLoadBal,dlb_scale,
mtop,box,ddbox,ir,dd,
cellsize_limit,cutoff_dd,
bInterCGBondeds,bInterCGMultiBody,
dd->nc);
}
else
{
limit = 0;
}
/* Communicate the information set by the master to all nodes */
gmx_bcast(sizeof(dd->nc),dd->nc,cr);
if (EEL_PME(ir->coulombtype))
{
gmx_bcast(sizeof(ir->nkx),&ir->nkx,cr);
gmx_bcast(sizeof(ir->nky),&ir->nky,cr);
gmx_bcast(sizeof(cr->npmenodes),&cr->npmenodes,cr);
}
else
{
cr->npmenodes = 0;
}
return limit;
}
