void write_espresso_conf_indexed(FILE *out, const char *title,
t_atoms *atoms, int nx, int *index,
rvec *x, rvec *v, matrix box)
{
int i, j;
fprintf(out, "# %s\n", title);
if (TRICLINIC(box))
{
gmx_warning("The Espresso format does not support triclinic unit-cells");
}
fprintf(out, "{variable {box_l %f %f %f}}\n", box[0][0], box[1][1], box[2][2]);
fprintf(out, "{particles {id pos type q%s}\n", v ? " v" : "");
for (i = 0; i < nx; i++)
{
if (index)
{
j = index[i];
}
else
{
j = i;
}
fprintf(out, "\t{%d %f %f %f %u %g",
j, x[j][XX], x[j][YY], x[j][ZZ],
atoms->atom[j].type, atoms->atom[j].q);
if (v)
{
fprintf(out, " %f %f %f", v[j][XX], v[j][YY], v[j][ZZ]);
}
fprintf(out, "}\n");
}
fprintf(out, "}\n");
}
int get_strings(const char *db,char ***strings)
{
FILE *in;
char **ptr;
char buf[256];
int i,nstr;
in=libopen(db);
if (fscanf(in,"%d",&nstr) != 1) {
gmx_warning("File %s is empty",db);
ffclose(in);
return 0;
}
snew(ptr,nstr);
for(i=0; (i<nstr); i++) {
if(1 != fscanf(in,"%s",buf))
{
gmx_fatal(FARGS,"Cannot read string from buffer");
}
#ifdef DEBUG
fprintf(stderr,"Have read: %s\n",buf);
#endif
ptr[i] = strdup(buf);
}
ffclose(in);
*strings=ptr;
return nstr;
}
int fget_lines(FILE *in,char ***strings)
{
char **ptr;
char buf[256];
int i,nstr;
char *pret;
pret = fgets(buf,255,in);
if ( pret==NULL || sscanf(buf,"%d",&nstr) != 1)
{
gmx_warning("File is empty");
ffclose(in);
return 0;
}
snew(ptr,nstr);
for(i=0; (i<nstr); i++) {
fgets2(buf,255,in);
ptr[i] = gmx_strdup(buf);
}
(*strings) = ptr;
return nstr;
}
static void enx_warning(const char *msg)
{
if (getenv("GMX_ENX_NO_FATAL") != NULL)
{
gmx_warning(msg);
}
else
{
gmx_fatal(FARGS,"%s\n%s",
msg,
"If you want to use the correct frames before the corrupted frame and avoid this fatal error set the env.var. GMX_ENX_NO_FATAL");
}
}
static void
nb_listed_warning_rlimit(const rvec *x, int ai, int aj, int * global_atom_index, real r, real rlimit)
{
gmx_warning("Listed nonbonded interaction between particles %d and %d\n"
"at distance %.3f which is larger than the table limit %.3f nm.\n\n"
"This is likely either a 1,4 interaction, or a listed interaction inside\n"
"a smaller molecule you are decoupling during a free energy calculation.\n"
"Since interactions at distances beyond the table cannot be computed,\n"
"they are skipped until they are inside the table limit again. You will\n"
"only see this message once, even if it occurs for several interactions.\n\n"
"IMPORTANT: This should not happen in a stable simulation, so there is\n"
"probably something wrong with your system. Only change the table-extension\n"
"distance in the mdp file if you are really sure that is the reason.\n",
glatnr(global_atom_index, ai), glatnr(global_atom_index, aj), r, rlimit);
if (debug)
{
fprintf(debug,
"%8f %8f %8f\n%8f %8f %8f\n1-4 (%d,%d) interaction not within cut-off! r=%g. Ignored\n",
x[ai][XX], x[ai][YY], x[ai][ZZ], x[aj][XX], x[aj][YY], x[aj][ZZ],
glatnr(global_atom_index, ai), glatnr(global_atom_index, aj), r);
}
}
static void atom_not_found(int fatal_errno, const char *file, int line,
const char *atomname, int resind,
const char *resname,
const char *bondtype, bool bAllowMissing)
{
char message_buffer[BUFSIZE];
if (strcmp(bondtype, "check") != 0)
{
if (0 != strcmp(bondtype, "atom"))
{
snprintf(message_buffer, 1024,
"Residue %d named %s of a molecule in the input file was mapped\n"
"to an entry in the topology database, but the atom %s used in\n"
"an interaction of type %s in that entry is not found in the\n"
"input file. Perhaps your atom and/or residue naming needs to be\n"
"fixed.\n",
resind+1, resname, atomname, bondtype);
}
else
{
snprintf(message_buffer, 1024,
"Residue %d named %s of a molecule in the input file was mapped\n"
"to an entry in the topology database, but the atom %s used in\n"
"that entry is not found in the input file. Perhaps your atom\n"
"and/or residue naming needs to be fixed.\n",
resind+1, resname, atomname);
}
if (bAllowMissing)
{
gmx_warning("WARNING: %s", message_buffer);
}
else
{
gmx_fatal(fatal_errno, file, line, "%s", message_buffer);
}
}
}
/*!
* Thread affinity set by the OpenMP library can conflict with the GROMACS
* internal affinity setting.
*
* While GNU OpenMP does not set affinity by default, the Intel OpenMP library
* does. This conflicts with the internal affinity (especially thread-MPI)
* setting, results in incorrectly locked threads, and causes dreadful performance.
*
* The KMP_AFFINITY environment variable is used by Intel, GOMP_CPU_AFFINITY
* by the GNU compilers (Intel also honors it well). If any of the variables
* is set, we honor it, disable the internal pinning, and warn the user.
* When using Intel OpenMP, we will disable affinity if the user did not set it
* anually through one of the aforementioned environment variables.
*
* Note that the Intel OpenMP affinity disabling iwll only take effect if this
* function is called before the OpenMP library gets initialized which happens
* when the first call is made into a compilation unit that contains OpenMP
* pragmas.
*/
void gmx_omp_check_thread_affinity(FILE *fplog, const t_commrec *cr,
gmx_hw_opt_t *hw_opt)
{
gmx_bool bKmpAffinitySet, bGompCpuAffinitySet;
char *kmp_env, *gomp_env;
/* no need to worry if internal thread pinning is turned off */
if (!hw_opt->bThreadPinning)
{
return;
}
#if defined(GMX_OPENMP)
/* We assume that the affinity setting is available on all platforms
* gcc supports. Even if this is not the case (e.g. Mac OS) the user
* will only get a warning.*/
bGompCpuAffinitySet = FALSE;
gomp_env = NULL;
#if defined(__GNUC__)
gomp_env = getenv("GOMP_CPU_AFFINITY");
bGompCpuAffinitySet = (gomp_env != NULL);
#endif /* __GNUC__ */
bKmpAffinitySet = FALSE;
#if defined(__INTEL_COMPILER)
kmp_env = getenv("KMP_AFFINITY");
bKmpAffinitySet = (kmp_env != NULL);
/* disable Intel OpenMP affinity if neither KMP_AFFINITY nor
* GOMP_CPU_AFFINITY is set (Intel uses the GNU env. var as well) */
if (!bKmpAffinitySet && !bGompCpuAffinitySet)
{
int retval;
#ifdef _MSC_VER
/* Windows not POSIX */
retval = _putenv_s("KMP_AFFINITY", "disabled");
#else
/* POSIX */
retval = setenv("KMP_AFFINITY", "disabled", 0);
#endif /* _MSC_VER */
if (debug)
{
fprintf(debug, "Disabling Intel OpenMP affinity by setting the KMP_AFFINITY=disabled env. var.\n");
}
if (retval != 0)
{
gmx_warning("Disabling Intel OpenMp affinity setting failed!");
}
}
/* turn off internal pinning KMP_AFFINITY != "disabled" */
if (bKmpAffinitySet && (gmx_strncasecmp(kmp_env, "disabled", 8) != 0))
{
md_print_warn(cr, fplog, "WARNING: KMP_AFFINITY set, will turn off %s internal affinity\n"
" setting as the two can conflict and cause performance degradation.\n"
" To keep using the %s internal affinity setting, set the\n"
" KMP_AFFINITY=disabled environment variable.",
ShortProgram(), ShortProgram());
hw_opt->bThreadPinning = FALSE;
}
#endif /* __INTEL_COMPILER */
#if defined(__INTEL_COMPILER) || defined(__GNUC__)
/* turn off internal pinning f GOMP_CPU_AFFINITY is set & non-empty */
if (bGompCpuAffinitySet && gomp_env != NULL && gomp_env != '\0')
{
md_print_warn(cr, fplog,
"WARNING: GOMP_CPU_AFFINITY set, will turn off %s internal affinity\n"
" setting as the two can conflict and cause performance degradation.\n"
" To keep using the %s internal affinity setting, unset the\n"
" GOMP_CPU_AFFINITY environment variable.",
ShortProgram(), ShortProgram());
hw_opt->bThreadPinning = FALSE;
}
#endif /* __INTEL_COMPILER || __GNUC__ */
#endif /* GMX_OPENMP */
}
/* Try to increase nstlist when using the Verlet cut-off scheme */
static void increase_nstlist(FILE *fp, t_commrec *cr,
t_inputrec *ir, int nstlist_cmdline,
const gmx_mtop_t *mtop, matrix box,
gmx_bool bGPU)
{
float listfac_ok, listfac_max;
int nstlist_orig, nstlist_prev;
verletbuf_list_setup_t ls;
real rlistWithReferenceNstlist, rlist_inc, rlist_ok, rlist_max;
real rlist_new, rlist_prev;
size_t nstlist_ind = 0;
t_state state_tmp;
gmx_bool bBox, bDD, bCont;
const char *nstl_gpu = "\nFor optimal performance with a GPU nstlist (now %d) should be larger.\nThe optimum depends on your CPU and GPU resources.\nYou might want to try several nstlist values.\n";
const char *nve_err = "Can not increase nstlist because an NVE ensemble is used";
const char *vbd_err = "Can not increase nstlist because verlet-buffer-tolerance is not set or used";
const char *box_err = "Can not increase nstlist because the box is too small";
const char *dd_err = "Can not increase nstlist because of domain decomposition limitations";
char buf[STRLEN];
const float oneThird = 1.0f / 3.0f;
if (nstlist_cmdline <= 0)
{
if (ir->nstlist == 1)
{
/* The user probably set nstlist=1 for a reason,
* don't mess with the settings.
*/
return;
}
if (fp != NULL && bGPU && ir->nstlist < nstlist_try[0])
{
fprintf(fp, nstl_gpu, ir->nstlist);
}
nstlist_ind = 0;
while (nstlist_ind < NNSTL && ir->nstlist >= nstlist_try[nstlist_ind])
{
nstlist_ind++;
}
if (nstlist_ind == NNSTL)
{
/* There are no larger nstlist value to try */
return;
}
}
if (EI_MD(ir->eI) && ir->etc == etcNO)
{
if (MASTER(cr))
{
fprintf(stderr, "%s\n", nve_err);
}
if (fp != NULL)
{
fprintf(fp, "%s\n", nve_err);
}
return;
}
if (ir->verletbuf_tol == 0 && bGPU)
{
gmx_fatal(FARGS, "You are using an old tpr file with a GPU, please generate a new tpr file with an up to date version of grompp");
}
if (ir->verletbuf_tol < 0)
{
if (MASTER(cr))
{
fprintf(stderr, "%s\n", vbd_err);
}
if (fp != NULL)
{
fprintf(fp, "%s\n", vbd_err);
}
return;
}
if (bGPU)
{
listfac_ok = nbnxn_gpu_listfac_ok;
listfac_max = nbnxn_gpu_listfac_max;
}
else
{
listfac_ok = nbnxn_cpu_listfac_ok;
listfac_max = nbnxn_cpu_listfac_max;
}
nstlist_orig = ir->nstlist;
if (nstlist_cmdline > 0)
{
if (fp)
{
sprintf(buf, "Getting nstlist=%d from command line option",
nstlist_cmdline);
}
ir->nstlist = nstlist_cmdline;
}
verletbuf_get_list_setup(TRUE, bGPU, &ls);
/* Allow rlist to make the list a given factor larger than the list
* would be with the reference value for nstlist (10).
*/
nstlist_prev = ir->nstlist;
ir->nstlist = nbnxnReferenceNstlist;
calc_verlet_buffer_size(mtop, det(box), ir, -1, &ls, NULL,
&rlistWithReferenceNstlist);
ir->nstlist = nstlist_prev;
/* Determine the pair list size increase due to zero interactions */
rlist_inc = nbnxn_get_rlist_effective_inc(ls.cluster_size_j,
mtop->natoms/det(box));
rlist_ok = (rlistWithReferenceNstlist + rlist_inc)*pow(listfac_ok, oneThird) - rlist_inc;
rlist_max = (rlistWithReferenceNstlist + rlist_inc)*pow(listfac_max, oneThird) - rlist_inc;
if (debug)
{
fprintf(debug, "nstlist tuning: rlist_inc %.3f rlist_ok %.3f rlist_max %.3f\n",
rlist_inc, rlist_ok, rlist_max);
}
nstlist_prev = nstlist_orig;
rlist_prev = ir->rlist;
do
{
if (nstlist_cmdline <= 0)
{
ir->nstlist = nstlist_try[nstlist_ind];
}
/* Set the pair-list buffer size in ir */
calc_verlet_buffer_size(mtop, det(box), ir, -1, &ls, NULL, &rlist_new);
/* Does rlist fit in the box? */
bBox = (sqr(rlist_new) < max_cutoff2(ir->ePBC, box));
bDD = TRUE;
if (bBox && DOMAINDECOMP(cr))
{
/* Check if rlist fits in the domain decomposition */
if (inputrec2nboundeddim(ir) < DIM)
{
gmx_incons("Changing nstlist with domain decomposition and unbounded dimensions is not implemented yet");
}
copy_mat(box, state_tmp.box);
bDD = change_dd_cutoff(cr, &state_tmp, ir, rlist_new);
}
if (debug)
{
fprintf(debug, "nstlist %d rlist %.3f bBox %d bDD %d\n",
ir->nstlist, rlist_new, bBox, bDD);
}
bCont = FALSE;
if (nstlist_cmdline <= 0)
{
if (bBox && bDD && rlist_new <= rlist_max)
{
/* Increase nstlist */
nstlist_prev = ir->nstlist;
rlist_prev = rlist_new;
bCont = (nstlist_ind+1 < NNSTL && rlist_new < rlist_ok);
}
else
{
/* Stick with the previous nstlist */
ir->nstlist = nstlist_prev;
rlist_new = rlist_prev;
bBox = TRUE;
bDD = TRUE;
}
}
nstlist_ind++;
}
while (bCont);
if (!bBox || !bDD)
{
gmx_warning(!bBox ? box_err : dd_err);
if (fp != NULL)
{
fprintf(fp, "\n%s\n", bBox ? box_err : dd_err);
}
ir->nstlist = nstlist_orig;
}
else if (ir->nstlist != nstlist_orig || rlist_new != ir->rlist)
{
sprintf(buf, "Changing nstlist from %d to %d, rlist from %g to %g",
nstlist_orig, ir->nstlist,
ir->rlist, rlist_new);
if (MASTER(cr))
{
fprintf(stderr, "%s\n\n", buf);
}
if (fp != NULL)
{
fprintf(fp, "%s\n\n", buf);
}
ir->rlist = rlist_new;
ir->rlistlong = rlist_new;
}
}
static void process_multiprot_output(const char *fn, real *rmsd, int *nres, rvec rotangles,
rvec translation, bool bCountres, t_countres *countres)
{
FILE *mpoutput;
char line[256];
char *string;
int i=0,j=0,res;
(*rmsd)=-1;
(*nres)=0;
mpoutput=gmx_ffopen (fn,"r");
if (bCountres) {
do {
fgets(line, 256, mpoutput);
} while (strstr(line,"Match List") == NULL);
fgets(line, 256, mpoutput);
do {
string = strtok (line,".");
while (i<2 && string != NULL) {
string = strtok (NULL,". ");
i++;
}
i=0;
res=atoi(string);
if (res > 0) {
while (countres[j].resnr!=res)
j++;
countres[j].count++;
}
fgets(line, 256, mpoutput);
} while (strstr(line,"End of Match List") == NULL);
rewind(mpoutput);
}
do {
fgets(line, 256, mpoutput);
} while (strstr(line,"Trans : ") == NULL);
string = strtok (line," :");
string = strtok (NULL," ");
while (i<3 && string != NULL) {
string = strtok (NULL," ");
rotangles[i]=atof(string);
i++;
}
i=0;
while (i<3 && string != NULL) {
string = strtok (NULL," ");
translation[i]=atof(string)/10;
i++;
}
if (i!=3) {
gmx_warning("Not enough values for rotation and translation vectors in the output of multiprot");
}
rotangles[YY]=rotangles[YY]*(-1);
while ((*rmsd) <0) {
fgets(line, 256, mpoutput);
if (strstr(line,"RMSD : ") != NULL) {
string = strtok (line,":");
string = strtok (NULL,":");
(*rmsd)=atof(string)/10;
}
}
while (!(*nres)) {
fgets(line,256, mpoutput);
if (strstr(line,"Match List") != NULL) {
string = strtok (line,":");
string = strtok (NULL,":");
(*nres) = atoi(string);
}
}
gmx_ffclose(mpoutput);
}
static gmx_bool get_w_conf(FILE *in, const char *infile, char *title,
t_symtab *symtab, t_atoms *atoms, int *ndec,
rvec x[], rvec *v, matrix box)
{
char name[6];
char line[STRLEN+1], *ptr;
char buf[256];
double x1, y1, z1, x2, y2, z2;
rvec xmin, xmax;
int natoms, i, m, resnr, newres, oldres, ddist, c;
gmx_bool bFirst, bVel;
char *p1, *p2, *p3;
newres = -1;
oldres = NOTSET; /* Unlikely number for the first residue! */
ddist = 0;
/* Read the title and number of atoms */
get_coordnum_fp(in, title, &natoms);
if (natoms > atoms->nr)
{
gmx_fatal(FARGS, "gro file contains more atoms (%d) than expected (%d)",
natoms, atoms->nr);
}
else if (natoms < atoms->nr)
{
fprintf(stderr, "Warning: gro file contains less atoms (%d) than expected"
" (%d)\n", natoms, atoms->nr);
}
bFirst = TRUE;
bVel = FALSE;
/* just pray the arrays are big enough */
for (i = 0; (i < natoms); i++)
{
if ((fgets2 (line, STRLEN, in)) == NULL)
{
gmx_fatal(FARGS, "Unexpected end of file in file %s at line %d",
infile, i+2);
}
if (strlen(line) < 39)
{
gmx_fatal(FARGS, "Invalid line in %s for atom %d:\n%s", infile, i+1, line);
}
/* determine read precision from distance between periods
(decimal points) */
if (bFirst)
{
bFirst = FALSE;
p1 = strchr(line, '.');
if (p1 == NULL)
{
gmx_fatal(FARGS, "A coordinate in file %s does not contain a '.'", infile);
}
p2 = strchr(&p1[1], '.');
if (p2 == NULL)
{
gmx_fatal(FARGS, "A coordinate in file %s does not contain a '.'", infile);
}
ddist = p2 - p1;
*ndec = ddist - 5;
p3 = strchr(&p2[1], '.');
if (p3 == NULL)
{
gmx_fatal(FARGS, "A coordinate in file %s does not contain a '.'", infile);
}
if (p3 - p2 != ddist)
{
gmx_fatal(FARGS, "The spacing of the decimal points in file %s is not consistent for x, y and z", infile);
}
}
/* residue number*/
memcpy(name, line, 5);
name[5] = '\0';
sscanf(name, "%d", &resnr);
memcpy(name, line+5, 5);
name[5] = '\0';
if (resnr != oldres)
{
oldres = resnr;
newres++;
if (newres >= natoms)
{
gmx_fatal(FARGS, "More residues than atoms in %s (natoms = %d)",
infile, natoms);
}
atoms->atom[i].resind = newres;
t_atoms_set_resinfo(atoms, i, symtab, name, resnr, ' ', 0, ' ');
}
else
{
atoms->atom[i].resind = newres;
}
/* atomname */
memcpy(name, line+10, 5);
atoms->atomname[i] = put_symtab(symtab, name);
/* eventueel controle atomnumber met i+1 */
/* coordinates (start after residue data) */
ptr = line + 20;
/* Read fixed format */
for (m = 0; m < DIM; m++)
{
for (c = 0; (c < ddist && ptr[0]); c++)
{
buf[c] = ptr[0];
ptr++;
}
buf[c] = '\0';
if (sscanf (buf, "%lf %lf", &x1, &x2) != 1)
{
gmx_fatal(FARGS, "Something is wrong in the coordinate formatting of file %s. Note that gro is fixed format (see the manual)", infile);
}
else
{
x[i][m] = x1;
}
}
/* velocities (start after residues and coordinates) */
if (v)
{
/* Read fixed format */
for (m = 0; m < DIM; m++)
{
for (c = 0; (c < ddist && ptr[0]); c++)
{
buf[c] = ptr[0];
ptr++;
}
buf[c] = '\0';
if (sscanf (buf, "%lf", &x1) != 1)
{
v[i][m] = 0;
}
else
{
v[i][m] = x1;
bVel = TRUE;
}
}
}
}
atoms->nres = newres + 1;
/* box */
fgets2 (line, STRLEN, in);
if (sscanf (line, "%lf%lf%lf", &x1, &y1, &z1) != 3)
{
gmx_warning("Bad box in file %s", infile);
/* Generate a cubic box */
for (m = 0; (m < DIM); m++)
{
xmin[m] = xmax[m] = x[0][m];
}
for (i = 1; (i < atoms->nr); i++)
{
for (m = 0; (m < DIM); m++)
{
xmin[m] = min(xmin[m], x[i][m]);
xmax[m] = max(xmax[m], x[i][m]);
}
}
for (i = 0; i < DIM; i++)
{
for (m = 0; m < DIM; m++)
{
box[i][m] = 0.0;
}
}
for (m = 0; (m < DIM); m++)
{
box[m][m] = (xmax[m]-xmin[m]);
}
fprintf(stderr, "Generated a cubic box %8.3f x %8.3f x %8.3f\n",
box[XX][XX], box[YY][YY], box[ZZ][ZZ]);
}
else
{
/* We found the first three values, the diagonal elements */
box[XX][XX] = x1;
box[YY][YY] = y1;
box[ZZ][ZZ] = z1;
if (sscanf (line, "%*f%*f%*f%lf%lf%lf%lf%lf%lf",
&x1, &y1, &z1, &x2, &y2, &z2) != 6)
{
x1 = y1 = z1 = x2 = y2 = z2 = 0.0;
}
box[XX][YY] = x1;
box[XX][ZZ] = y1;
box[YY][XX] = z1;
box[YY][ZZ] = x2;
box[ZZ][XX] = y2;
box[ZZ][YY] = z2;
}
return bVel;
}
void set_pdb_conf_bfac(int natoms, int nres, t_atoms *atoms, int n_bfac,
double *bfac, int *bfac_nr, gmx_bool peratom)
{
FILE *out;
real bfac_min, bfac_max;
int i, n;
gmx_bool found;
bfac_max = -1e10;
bfac_min = 1e10;
for (i = 0; (i < n_bfac); i++)
{
if (bfac_nr[i] - 1 >= atoms->nres)
peratom = TRUE;
/* if ((bfac_nr[i]-1<0) || (bfac_nr[i]-1>=atoms->nr))
gmx_fatal(FARGS,"Index of B-Factor %d is out of range: %d (%g)",
i+1,bfac_nr[i],bfac[i]); */
if (bfac[i] > bfac_max)
bfac_max = bfac[i];
if (bfac[i] < bfac_min)
bfac_min = bfac[i];
}
while ((bfac_max > 99.99) || (bfac_min < -99.99))
{
fprintf(stderr,
"Range of values for B-factors too large (min %g, max %g) "
"will scale down a factor 10\n", bfac_min, bfac_max);
for (i = 0; (i < n_bfac); i++)
bfac[i] /= 10;
bfac_max /= 10;
bfac_min /= 10;
}
while ((fabs(bfac_max) < 0.5) && (fabs(bfac_min) < 0.5))
{
fprintf(stderr,
"Range of values for B-factors too small (min %g, max %g) "
"will scale up a factor 10\n", bfac_min, bfac_max);
for (i = 0; (i < n_bfac); i++)
bfac[i] *= 10;
bfac_max *= 10;
bfac_min *= 10;
}
for (i = 0; (i < natoms); i++)
atoms->pdbinfo[i].bfac = 0;
if (!peratom)
{
fprintf(stderr, "Will attach %d B-factors to %d residues\n", n_bfac,
nres);
for (i = 0; (i < n_bfac); i++)
{
found = FALSE;
for (n = 0; (n < natoms); n++)
if (bfac_nr[i] == atoms->resinfo[atoms->atom[n].resind].nr)
{
atoms->pdbinfo[n].bfac = bfac[i];
found = TRUE;
}
if (!found)
{
gmx_warning("Residue nr %d not found\n", bfac_nr[i]);
}
}
}
else
{
fprintf(stderr, "Will attach %d B-factors to %d atoms\n", n_bfac,
natoms);
for (i = 0; (i < n_bfac); i++)
{
atoms->pdbinfo[bfac_nr[i] - 1].bfac = bfac[i];
}
}
}
static void do_sham(const char *fn, const char *ndx,
const char *xpmP, const char *xpm, const char *xpm2,
const char *xpm3, const char *pdb,
const char *logf,
int n, int neig, real **eig,
gmx_bool bGE, int nenerT, real **enerT,
real Tref,
real pmax, real gmax,
real *emin, real *emax, int nlevels, real pmin,
int *idim, int *ibox,
gmx_bool bXmin, real *xmin, gmx_bool bXmax, real *xmax)
{
FILE *fp;
real *min_eig, *max_eig;
real *axis_x, *axis_y, *axis_z, *axis = NULL;
double *P;
real **PP, *W, *E, **WW, **EE, *S, **SS, *M, *bE;
rvec xxx;
char *buf;
double *bfac, efac, bref, Pmax, Wmin, Wmax, Winf, Emin, Emax, Einf, Smin, Smax, Sinf, Mmin, Mmax;
real *delta;
int i, j, k, imin, len, index, d, *nbin, *bindex, bi;
int *nxyz, maxbox;
t_blocka *b;
gmx_bool bOutside;
unsigned int flags;
t_rgb rlo = { 0, 0, 0 };
t_rgb rhi = { 1, 1, 1 };
/* Determine extremes for the eigenvectors */
snew(min_eig, neig);
snew(max_eig, neig);
snew(nxyz, neig);
snew(bfac, neig);
snew(delta, neig);
for (i = 0; (i < neig); i++)
{
/* Check for input constraints */
min_eig[i] = max_eig[i] = eig[i][0];
for (j = 0; (j < n); j++)
{
min_eig[i] = min(min_eig[i], eig[i][j]);
max_eig[i] = max(max_eig[i], eig[i][j]);
delta[i] = (max_eig[i]-min_eig[i])/(2.0*ibox[i]);
}
/* Add some extra space, half a bin on each side, unless the
* user has set the limits.
*/
if (bXmax)
{
if (max_eig[i] > xmax[i])
{
gmx_warning("Your xmax[%d] value %f is smaller than the largest data point %f", i, xmax[i], max_eig[i]);
}
max_eig[i] = xmax[i];
}
else
{
max_eig[i] += delta[i];
}
if (bXmin)
{
if (min_eig[i] < xmin[i])
{
gmx_warning("Your xmin[%d] value %f is larger than the smallest data point %f", i, xmin[i], min_eig[i]);
}
min_eig[i] = xmin[i];
}
else
{
min_eig[i] -= delta[i];
}
bfac[i] = ibox[i]/(max_eig[i]-min_eig[i]);
}
/* Do the binning */
bref = 1/(BOLTZ*Tref);
snew(bE, n);
if (bGE || nenerT == 2)
{
Emin = 1e8;
for (j = 0; (j < n); j++)
{
if (bGE)
{
bE[j] = bref*enerT[0][j];
}
else
{
bE[j] = (bref - 1/(BOLTZ*enerT[1][j]))*enerT[0][j];
}
Emin = min(Emin, bE[j]);
}
}
else
{
Emin = 0;
}
len = 1;
for (i = 0; (i < neig); i++)
{
len = len*ibox[i];
}
printf("There are %d bins in the %d-dimensional histogram. Beta-Emin = %g\n",
len, neig, Emin);
snew(P, len);
snew(W, len);
snew(E, len);
snew(S, len);
snew(M, len);
snew(nbin, len);
snew(bindex, n);
/* Loop over projections */
for (j = 0; (j < n); j++)
{
/* Loop over dimensions */
bOutside = FALSE;
for (i = 0; (i < neig); i++)
{
nxyz[i] = bfac[i]*(eig[i][j]-min_eig[i]);
if (nxyz[i] < 0 || nxyz[i] >= ibox[i])
{
bOutside = TRUE;
}
}
if (!bOutside)
{
index = indexn(neig, ibox, nxyz);
range_check(index, 0, len);
/* Compute the exponential factor */
if (enerT)
{
efac = exp(-bE[j]+Emin);
}
else
{
efac = 1;
}
/* Apply the bin volume correction for a multi-dimensional distance */
for (i = 0; i < neig; i++)
{
if (idim[i] == 2)
{
efac /= eig[i][j];
}
else if (idim[i] == 3)
{
efac /= sqr(eig[i][j]);
}
else if (idim[i] == -1)
{
efac /= sin(DEG2RAD*eig[i][j]);
}
}
/* Update the probability */
P[index] += efac;
/* Update the energy */
if (enerT)
{
E[index] += enerT[0][j];
}
/* Statistics: which "structure" in which bin */
nbin[index]++;
bindex[j] = index;
}
}
/* Normalize probability */
normalize_p_e(len, P, nbin, E, pmin);
Pmax = 0;
/* Compute boundaries for the Free energy */
Wmin = 1e8;
imin = -1;
Wmax = -1e8;
/* Recompute Emin: it may have changed due to averaging */
Emin = 1e8;
Emax = -1e8;
for (i = 0; (i < len); i++)
{
if (P[i] != 0)
{
Pmax = max(P[i], Pmax);
W[i] = -BOLTZ*Tref*log(P[i]);
if (W[i] < Wmin)
{
Wmin = W[i];
imin = i;
}
Emin = min(E[i], Emin);
Emax = max(E[i], Emax);
Wmax = max(W[i], Wmax);
}
}
if (pmax > 0)
{
Pmax = pmax;
}
if (gmax > 0)
{
Wmax = gmax;
}
else
{
Wmax -= Wmin;
}
Winf = Wmax+1;
Einf = Emax+1;
Smin = Emin-Wmax;
Smax = Emax-Smin;
Sinf = Smax+1;
/* Write out the free energy as a function of bin index */
fp = gmx_ffopen(fn, "w");
for (i = 0; (i < len); i++)
{
if (P[i] != 0)
{
W[i] -= Wmin;
S[i] = E[i]-W[i]-Smin;
fprintf(fp, "%5d %10.5e %10.5e %10.5e\n", i, W[i], E[i], S[i]);
}
else
{
W[i] = Winf;
E[i] = Einf;
S[i] = Sinf;
}
}
gmx_ffclose(fp);
/* Organize the structures in the bins */
snew(b, 1);
snew(b->index, len+1);
snew(b->a, n);
b->index[0] = 0;
for (i = 0; (i < len); i++)
{
b->index[i+1] = b->index[i]+nbin[i];
nbin[i] = 0;
}
for (i = 0; (i < n); i++)
{
bi = bindex[i];
b->a[b->index[bi]+nbin[bi]] = i;
nbin[bi]++;
}
/* Consistency check */
/* This no longer applies when we allow the plot to be smaller
than the sampled space.
for(i=0; (i<len); i++) {
if (nbin[i] != (b->index[i+1] - b->index[i]))
gmx_fatal(FARGS,"nbin[%d] = %d, should be %d",i,nbin[i],
b->index[i+1] - b->index[i]);
}
*/
/* Write the index file */
fp = gmx_ffopen(ndx, "w");
for (i = 0; (i < len); i++)
{
if (nbin[i] > 0)
{
fprintf(fp, "[ %d ]\n", i);
for (j = b->index[i]; (j < b->index[i+1]); j++)
{
fprintf(fp, "%d\n", b->a[j]+1);
}
}
}
gmx_ffclose(fp);
snew(axis_x, ibox[0]+1);
snew(axis_y, ibox[1]+1);
snew(axis_z, ibox[2]+1);
maxbox = max(ibox[0], max(ibox[1], ibox[2]));
snew(PP, maxbox*maxbox);
snew(WW, maxbox*maxbox);
snew(EE, maxbox*maxbox);
snew(SS, maxbox*maxbox);
for (i = 0; (i < min(neig, 3)); i++)
{
switch (i)
{
case 0: axis = axis_x; break;
case 1: axis = axis_y; break;
case 2: axis = axis_z; break;
default: break;
}
for (j = 0; j <= ibox[i]; j++)
{
axis[j] = min_eig[i] + j/bfac[i];
}
}
pick_minima(logf, ibox, neig, len, W);
if (gmax <= 0)
{
gmax = Winf;
}
flags = MAT_SPATIAL_X | MAT_SPATIAL_Y;
if (neig == 2)
{
/* Dump to XPM file */
snew(PP, ibox[0]);
for (i = 0; (i < ibox[0]); i++)
{
snew(PP[i], ibox[1]);
for (j = 0; j < ibox[1]; j++)
{
PP[i][j] = P[i*ibox[1]+j];
}
WW[i] = &(W[i*ibox[1]]);
EE[i] = &(E[i*ibox[1]]);
SS[i] = &(S[i*ibox[1]]);
}
fp = gmx_ffopen(xpmP, "w");
write_xpm(fp, flags, "Probability Distribution", "", "PC1", "PC2",
ibox[0], ibox[1], axis_x, axis_y, PP, 0, Pmax, rlo, rhi, &nlevels);
gmx_ffclose(fp);
fp = gmx_ffopen(xpm, "w");
write_xpm(fp, flags, "Gibbs Energy Landscape", "G (kJ/mol)", "PC1", "PC2",
ibox[0], ibox[1], axis_x, axis_y, WW, 0, gmax, rlo, rhi, &nlevels);
gmx_ffclose(fp);
fp = gmx_ffopen(xpm2, "w");
write_xpm(fp, flags, "Enthalpy Landscape", "H (kJ/mol)", "PC1", "PC2",
ibox[0], ibox[1], axis_x, axis_y, EE,
emin ? *emin : Emin, emax ? *emax : Einf, rlo, rhi, &nlevels);
gmx_ffclose(fp);
fp = gmx_ffopen(xpm3, "w");
write_xpm(fp, flags, "Entropy Landscape", "TDS (kJ/mol)", "PC1", "PC2",
ibox[0], ibox[1], axis_x, axis_y, SS, 0, Sinf, rlo, rhi, &nlevels);
gmx_ffclose(fp);
}
else if (neig == 3)
{
/* Dump to PDB file */
fp = gmx_ffopen(pdb, "w");
for (i = 0; (i < ibox[0]); i++)
{
xxx[XX] = 3*(i+0.5-ibox[0]/2);
for (j = 0; (j < ibox[1]); j++)
{
xxx[YY] = 3*(j+0.5-ibox[1]/2);
for (k = 0; (k < ibox[2]); k++)
{
xxx[ZZ] = 3*(k+0.5-ibox[2]/2);
index = index3(ibox, i, j, k);
if (P[index] > 0)
{
fprintf(fp, "%-6s%5u %-4.4s%3.3s %4d %8.3f%8.3f%8.3f%6.2f%6.2f\n",
"ATOM", (index+1) %10000, "H", "H", (index+1)%10000,
xxx[XX], xxx[YY], xxx[ZZ], 1.0, W[index]);
}
}
}
}
gmx_ffclose(fp);
write_xplor("out.xplor", W, ibox, min_eig, max_eig);
nxyz[XX] = imin/(ibox[1]*ibox[2]);
nxyz[YY] = (imin-nxyz[XX]*ibox[1]*ibox[2])/ibox[2];
nxyz[ZZ] = imin % ibox[2];
for (i = 0; (i < ibox[0]); i++)
{
snew(WW[i], maxbox);
for (j = 0; (j < ibox[1]); j++)
{
WW[i][j] = W[index3(ibox, i, j, nxyz[ZZ])];
}
}
snew(buf, strlen(xpm)+4);
sprintf(buf, "%s", xpm);
sprintf(&buf[strlen(xpm)-4], "12.xpm");
fp = gmx_ffopen(buf, "w");
write_xpm(fp, flags, "Gibbs Energy Landscape", "W (kJ/mol)", "PC1", "PC2",
ibox[0], ibox[1], axis_x, axis_y, WW, 0, gmax, rlo, rhi, &nlevels);
gmx_ffclose(fp);
for (i = 0; (i < ibox[0]); i++)
{
for (j = 0; (j < ibox[2]); j++)
{
WW[i][j] = W[index3(ibox, i, nxyz[YY], j)];
}
}
sprintf(&buf[strlen(xpm)-4], "13.xpm");
fp = gmx_ffopen(buf, "w");
write_xpm(fp, flags, "SHAM Energy Landscape", "kJ/mol", "PC1", "PC3",
ibox[0], ibox[2], axis_x, axis_z, WW, 0, gmax, rlo, rhi, &nlevels);
gmx_ffclose(fp);
for (i = 0; (i < ibox[1]); i++)
{
for (j = 0; (j < ibox[2]); j++)
{
WW[i][j] = W[index3(ibox, nxyz[XX], i, j)];
}
}
sprintf(&buf[strlen(xpm)-4], "23.xpm");
fp = gmx_ffopen(buf, "w");
write_xpm(fp, flags, "SHAM Energy Landscape", "kJ/mol", "PC2", "PC3",
ibox[1], ibox[2], axis_y, axis_z, WW, 0, gmax, rlo, rhi, &nlevels);
gmx_ffclose(fp);
sfree(buf);
}
}
/** Determine the number of threads for module \p mod.
*
* \p m takes values form the module_nth_t enum and maps these to the
* corresponding value in modth_env_var.
*
* Each number of threads per module takes the default value unless
* GMX_*_NUM_THERADS env var is set, case in which its value overrides
* the deafult.
*
* The "group" scheme supports OpenMP only in PME and in thise case all but
* the PME nthread values default to 1.
*/
static void pick_module_nthreads(FILE *fplog, int m,
gmx_bool bSimMaster,
gmx_bool bFullOmpSupport,
gmx_bool bSepPME)
{
char *env;
int nth;
char sbuf[STRLEN];
gmx_bool bOMP;
#ifdef GMX_OPENMP
bOMP = TRUE;
#else
bOMP = FALSE;
#endif /* GMX_OPENMP */
/* The default should never be set through a GMX_*_NUM_THREADS env var
* as it's always equal with gnth. */
if (m == emntDefault)
{
return;
}
/* check the environment variable */
if ((env = getenv(modth_env_var[m])) != NULL)
{
sscanf(env, "%d", &nth);
if (!bOMP)
{
gmx_warning("%s=%d is set, but %s is compiled without OpenMP!",
modth_env_var[m], nth, ShortProgram());
}
/* with the verlet codepath, when any GMX_*_NUM_THREADS env var is set,
* OMP_NUM_THREADS also has to be set */
if (bFullOmpSupport && getenv("OMP_NUM_THREADS") == NULL)
{
gmx_fatal(FARGS, "%s=%d is set, the default number of threads also "
"needs to be set with OMP_NUM_THREADS!",
modth_env_var[m], nth);
}
/* with the group scheme warn if any env var except PME is set */
if (!bFullOmpSupport)
{
if (m != emntPME)
{
gmx_warning("%s=%d is set, but OpenMP multithreading is not "
"supported in %s!",
modth_env_var[m], nth, mod_name[m]);
nth = 1;
}
}
/* only babble if we are really overriding with a different value */
if ((bSepPME && m == emntPME && nth != modth.gnth_pme) || (nth != modth.gnth))
{
sprintf(sbuf, "%s=%d set, overriding the default number of %s threads",
modth_env_var[m], nth, mod_name[m]);
if (bSimMaster)
{
fprintf(stderr, "\n%s\n", sbuf);
}
if (fplog)
{
fprintf(fplog, "%s\n", sbuf);
}
}
}
else
{
/* pick the global PME node nthreads if we are setting the number
* of threads in separate PME nodes */
nth = (bSepPME && m == emntPME) ? modth.gnth_pme : modth.gnth;
}
gmx_omp_nthreads_set(m, nth);
}
gmx_bool gmx_omp_check_thread_affinity(char **message)
{
bool shouldSetAffinity = true;
*message = NULL;
#ifdef GMX_OPENMP
/* We assume that the affinity setting is available on all platforms
* gcc supports. Even if this is not the case (e.g. Mac OS) the user
* will only get a warning. */
#if defined(__GNUC__) || defined(__INTEL_COMPILER)
const char *programName;
try
{
programName = gmx::getProgramContext().displayName();
}
GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
const char *const gomp_env = getenv("GOMP_CPU_AFFINITY");
const bool bGompCpuAffinitySet = (gomp_env != NULL);
/* turn off internal pinning if GOMP_CPU_AFFINITY is set & non-empty */
if (bGompCpuAffinitySet && *gomp_env != '\0')
{
try
{
std::string buf = gmx::formatString(
"NOTE: GOMP_CPU_AFFINITY set, will turn off %s internal affinity\n"
" setting as the two can conflict and cause performance degradation.\n"
" To keep using the %s internal affinity setting, unset the\n"
" GOMP_CPU_AFFINITY environment variable.",
programName, programName);
*message = gmx_strdup(buf.c_str());
}
GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
shouldSetAffinity = false;
}
#endif /* __GNUC__ || __INTEL_COMPILER */
#if defined(__INTEL_COMPILER)
const char *const kmp_env = getenv("KMP_AFFINITY");
const bool bKmpAffinitySet = (kmp_env != NULL);
/* disable Intel OpenMP affinity if neither KMP_AFFINITY nor
* GOMP_CPU_AFFINITY is set (Intel uses the GNU env. var as well) */
if (!bKmpAffinitySet && !bGompCpuAffinitySet)
{
int retval;
#ifdef _MSC_VER
/* Windows not POSIX */
retval = _putenv_s("KMP_AFFINITY", "disabled");
#else
/* POSIX */
retval = setenv("KMP_AFFINITY", "disabled", 0);
#endif /* _MSC_VER */
if (debug)
{
fprintf(debug, "Disabling Intel OpenMP affinity by setting the KMP_AFFINITY=disabled env. var.\n");
}
if (retval != 0)
{
gmx_warning("Disabling Intel OpenMp affinity setting failed!");
}
}
/* turn off internal pinning KMP_AFFINITY != "disabled" */
if (bKmpAffinitySet && (gmx_strncasecmp(kmp_env, "disabled", 8) != 0))
{
try
{
std::string buf = gmx::formatString(
"NOTE: KMP_AFFINITY set, will turn off %s internal affinity\n"
" setting as the two can conflict and cause performance degradation.\n"
" To keep using the %s internal affinity setting, set the\n"
" KMP_AFFINITY=disabled environment variable.",
programName, programName);
*message = gmx_strdup(buf.c_str());
}
GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR;
shouldSetAffinity = false;
}
#endif /* __INTEL_COMPILER */
#endif /* GMX_OPENMP */
return shouldSetAffinity;
}
/* TODO: If/when TNG acquires the ability to copy data blocks without
* uncompressing them, then this implemenation should be reconsidered.
* Ideally, gmx trjconv -f a.tng -o b.tng -b 10 -e 20 would be fast
* and lose no information. */
gmx_bool gmx_read_next_tng_frame(tng_trajectory_t input,
t_trxframe *fr,
gmx_int64_t *requestedIds,
int numRequestedIds)
{
#if GMX_USE_TNG
gmx_bool bOK = TRUE;
tng_function_status stat;
gmx_int64_t numberOfAtoms = -1, frameNumber = -1;
gmx_int64_t nBlocks, blockId, *blockIds = NULL, codecId;
char datatype = -1;
void *values = NULL;
double frameTime = -1.0;
int size, blockDependency;
double prec;
const int defaultNumIds = 5;
static gmx_int64_t fallbackRequestedIds[defaultNumIds] =
{
TNG_TRAJ_BOX_SHAPE, TNG_TRAJ_POSITIONS,
TNG_TRAJ_VELOCITIES, TNG_TRAJ_FORCES,
TNG_GMX_LAMBDA
};
fr->bStep = FALSE;
fr->bTime = FALSE;
fr->bLambda = FALSE;
fr->bAtoms = FALSE;
fr->bPrec = FALSE;
fr->bX = FALSE;
fr->bV = FALSE;
fr->bF = FALSE;
fr->bBox = FALSE;
/* If no specific IDs were requested read all block types that can
* currently be interpreted */
if (!requestedIds || numRequestedIds == 0)
{
numRequestedIds = defaultNumIds;
requestedIds = fallbackRequestedIds;
}
stat = tng_num_particles_get(input, &numberOfAtoms);
if (stat != TNG_SUCCESS)
{
gmx_file("Cannot determine number of atoms from TNG file.");
}
fr->natoms = numberOfAtoms;
if (!gmx_get_tng_data_block_types_of_next_frame(input,
fr->step,
numRequestedIds,
requestedIds,
&frameNumber,
&nBlocks,
&blockIds))
{
return FALSE;
}
if (nBlocks == 0)
{
return FALSE;
}
for (gmx_int64_t i = 0; i < nBlocks; i++)
{
blockId = blockIds[i];
tng_data_block_dependency_get(input, blockId, &blockDependency);
if (blockDependency & TNG_PARTICLE_DEPENDENT)
{
stat = tng_util_particle_data_next_frame_read(input,
blockId,
&values,
&datatype,
&frameNumber,
&frameTime);
}
else
{
stat = tng_util_non_particle_data_next_frame_read(input,
blockId,
&values,
&datatype,
&frameNumber,
&frameTime);
}
if (stat == TNG_CRITICAL)
{
gmx_file("Cannot read positions from TNG file.");
return FALSE;
}
else if (stat == TNG_FAILURE)
{
continue;
}
switch (blockId)
{
case TNG_TRAJ_BOX_SHAPE:
switch (datatype)
{
case TNG_INT_DATA:
size = sizeof(gmx_int64_t);
break;
case TNG_FLOAT_DATA:
size = sizeof(float);
break;
case TNG_DOUBLE_DATA:
size = sizeof(double);
break;
default:
gmx_incons("Illegal datatype of box shape values!");
}
for (int i = 0; i < DIM; i++)
{
convert_array_to_real_array(reinterpret_cast<char *>(values) + size * i * DIM,
reinterpret_cast<real *>(fr->box[i]),
getDistanceScaleFactor(input),
1,
DIM,
datatype);
}
fr->bBox = TRUE;
break;
case TNG_TRAJ_POSITIONS:
srenew(fr->x, fr->natoms);
convert_array_to_real_array(values,
reinterpret_cast<real *>(fr->x),
getDistanceScaleFactor(input),
fr->natoms,
DIM,
datatype);
fr->bX = TRUE;
tng_util_frame_current_compression_get(input, blockId, &codecId, &prec);
/* This must be updated if/when more lossy compression methods are added */
if (codecId == TNG_TNG_COMPRESSION)
{
fr->prec = prec;
fr->bPrec = TRUE;
}
break;
case TNG_TRAJ_VELOCITIES:
srenew(fr->v, fr->natoms);
convert_array_to_real_array(values,
(real *) fr->v,
getDistanceScaleFactor(input),
fr->natoms,
DIM,
datatype);
fr->bV = TRUE;
tng_util_frame_current_compression_get(input, blockId, &codecId, &prec);
/* This must be updated if/when more lossy compression methods are added */
if (codecId == TNG_TNG_COMPRESSION)
{
fr->prec = prec;
fr->bPrec = TRUE;
}
break;
case TNG_TRAJ_FORCES:
srenew(fr->f, fr->natoms);
convert_array_to_real_array(values,
reinterpret_cast<real *>(fr->f),
getDistanceScaleFactor(input),
fr->natoms,
DIM,
datatype);
fr->bF = TRUE;
break;
case TNG_GMX_LAMBDA:
switch (datatype)
{
case TNG_FLOAT_DATA:
fr->lambda = *(reinterpret_cast<float *>(values));
break;
case TNG_DOUBLE_DATA:
fr->lambda = *(reinterpret_cast<double *>(values));
break;
default:
gmx_incons("Illegal datatype lambda value!");
}
fr->bLambda = TRUE;
break;
default:
gmx_warning("Illegal block type! Currently GROMACS tools can only handle certain data types. Skipping block.");
}
/* values does not have to be freed before reading next frame. It will
* be reallocated if it is not NULL. */
}
fr->step = static_cast<int>(frameNumber);
fr->bStep = TRUE;
// Convert the time to ps
fr->time = frameTime / PICO;
fr->bTime = TRUE;
/* values must be freed before leaving this function */
sfree(values);
return bOK;
#else
GMX_UNUSED_VALUE(input);
GMX_UNUSED_VALUE(fr);
GMX_UNUSED_VALUE(requestedIds);
GMX_UNUSED_VALUE(numRequestedIds);
return FALSE;
#endif
}
/*! \brief Helper function for parsing various input about the number
of OpenMP threads to use in various modules and deciding what to
do about it. */
static void manage_number_of_openmp_threads(const gmx::MDLogger &mdlog,
const t_commrec *cr,
bool bOMP,
int nthreads_hw_avail,
int omp_nthreads_req,
int omp_nthreads_pme_req,
gmx_bool gmx_unused bThisNodePMEOnly,
gmx_bool bFullOmpSupport,
int numRanksOnThisNode,
gmx_bool bSepPME)
{
int nth;
char *env;
#if GMX_THREAD_MPI
/* modth is shared among tMPI threads, so for thread safety, the
* detection is done on the master only. It is not thread-safe
* with multiple simulations, but that's anyway not supported by
* tMPI. */
if (!SIMMASTER(cr))
{
return;
}
#else
GMX_UNUSED_VALUE(cr);
#endif
if (modth.initialized)
{
/* Just return if the initialization has already been
done. This could only happen if gmx_omp_nthreads_init() has
already been called. */
return;
}
/* With full OpenMP support (verlet scheme) set the number of threads
* per process / default:
* - 1 if not compiled with OpenMP or
* - OMP_NUM_THREADS if the env. var is set, or
* - omp_nthreads_req = #of threads requested by the user on the mdrun
* command line, otherwise
* - take the max number of available threads and distribute them
* on the processes/tMPI threads.
* ~ The GMX_*_NUM_THREADS env var overrides the number of threads of
* the respective module and it has to be used in conjunction with
* OMP_NUM_THREADS.
*
* With the group scheme OpenMP multithreading is only supported in PME,
* for all other modules nthreads is set to 1.
* The number of PME threads is equal to:
* - 1 if not compiled with OpenMP or
* - GMX_PME_NUM_THREADS if defined, otherwise
* - OMP_NUM_THREADS if defined, otherwise
* - 1
*/
nth = 1;
if ((env = getenv("OMP_NUM_THREADS")) != nullptr)
{
if (!bOMP && (std::strncmp(env, "1", 1) != 0))
{
gmx_warning("OMP_NUM_THREADS is set, but %s was compiled without OpenMP support!",
gmx::getProgramContext().displayName());
}
else
{
nth = gmx_omp_get_max_threads();
}
}
else if (omp_nthreads_req > 0)
{
nth = omp_nthreads_req;
}
else if (bFullOmpSupport && bOMP)
{
/* max available threads per node */
nth = nthreads_hw_avail;
/* divide the threads among the MPI ranks */
if (nth >= numRanksOnThisNode)
{
nth /= numRanksOnThisNode;
}
else
{
nth = 1;
}
}
/* now we have the global values, set them:
* - 1 if not compiled with OpenMP and for the group scheme
* - nth for the verlet scheme when compiled with OpenMP
*/
if (bFullOmpSupport && bOMP)
{
modth.gnth = nth;
}
else
{
modth.gnth = 1;
}
if (bSepPME)
{
if (omp_nthreads_pme_req > 0)
{
modth.gnth_pme = omp_nthreads_pme_req;
}
else
{
modth.gnth_pme = nth;
}
}
else
{
modth.gnth_pme = 0;
}
/* now set the per-module values */
modth.nth[emntDefault] = modth.gnth;
pick_module_nthreads(mdlog, emntDomdec, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntPairsearch, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntNonbonded, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntBonded, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntPME, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntUpdate, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntVSITE, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntLINCS, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntSETTLE, bFullOmpSupport, bSepPME);
/* set the number of threads globally */
if (bOMP)
{
#if !GMX_THREAD_MPI
if (bThisNodePMEOnly)
{
gmx_omp_set_num_threads(modth.gnth_pme);
}
else
#endif /* GMX_THREAD_MPI */
{
if (bFullOmpSupport)
{
gmx_omp_set_num_threads(nth);
}
else
{
gmx_omp_set_num_threads(1);
}
}
}
modth.initialized = TRUE;
}
/** Determine the number of threads for module \p mod.
*
* \p m takes values form the module_nth_t enum and maps these to the
* corresponding value in modth_env_var.
*
* Each number of threads per module takes the default value unless
* GMX_*_NUM_THERADS env var is set, case in which its value overrides
* the deafult.
*
* The "group" scheme supports OpenMP only in PME and in thise case all but
* the PME nthread values default to 1.
*/
static void pick_module_nthreads(const gmx::MDLogger &mdlog, int m,
gmx_bool bFullOmpSupport,
gmx_bool bSepPME)
{
char *env;
int nth;
const bool bOMP = GMX_OPENMP;
/* The default should never be set through a GMX_*_NUM_THREADS env var
* as it's always equal with gnth. */
if (m == emntDefault)
{
return;
}
/* check the environment variable */
if ((env = getenv(modth_env_var[m])) != nullptr)
{
sscanf(env, "%d", &nth);
if (!bOMP)
{
gmx_warning("%s=%d is set, but %s is compiled without OpenMP!",
modth_env_var[m], nth,
gmx::getProgramContext().displayName());
}
/* with the verlet codepath, when any GMX_*_NUM_THREADS env var is set,
* OMP_NUM_THREADS also has to be set */
if (bFullOmpSupport && getenv("OMP_NUM_THREADS") == nullptr)
{
gmx_warning("%s=%d is set, the default number of threads also "
"needs to be set with OMP_NUM_THREADS!",
modth_env_var[m], nth);
}
/* with the group scheme warn if any env var except PME is set */
if (!bFullOmpSupport)
{
if (m != emntPME)
{
gmx_warning("%s=%d is set, but OpenMP multithreading is not "
"supported in %s!",
modth_env_var[m], nth, mod_name[m]);
nth = 1;
}
}
/* only babble if we are really overriding with a different value */
if ((bSepPME && m == emntPME && nth != modth.gnth_pme) || (nth != modth.gnth))
{
GMX_LOG(mdlog.warning).asParagraph().appendTextFormatted(
"%s=%d set, overriding the default number of %s threads",
modth_env_var[m], nth, mod_name[m]);
}
}
else
{
/* pick the global PME node nthreads if we are setting the number
* of threads in separate PME nodes */
nth = (bSepPME && m == emntPME) ? modth.gnth_pme : modth.gnth;
}
gmx_omp_nthreads_set(m, nth);
}
void gmx_omp_nthreads_init(FILE *fplog, t_commrec *cr,
int nthreads_hw_avail,
int omp_nthreads_req,
int omp_nthreads_pme_req,
gmx_bool gmx_unused bThisNodePMEOnly,
gmx_bool bFullOmpSupport)
{
int nth, nth_pmeonly, gmx_maxth, nppn;
char *env;
gmx_bool bSepPME, bOMP;
#ifdef GMX_OPENMP
bOMP = TRUE;
#else
bOMP = FALSE;
#endif /* GMX_OPENMP */
/* number of MPI processes/threads per physical node */
nppn = cr->nrank_intranode;
bSepPME = ( (cr->duty & DUTY_PP) && !(cr->duty & DUTY_PME)) ||
(!(cr->duty & DUTY_PP) && (cr->duty & DUTY_PME));
#ifdef GMX_THREAD_MPI
/* modth is shared among tMPI threads, so for thread safety do the
* detection is done on the master only. It is not thread-safe with
* multiple simulations, but that's anyway not supported by tMPI. */
if (SIMMASTER(cr))
#endif
{
/* just return if the initialization has already been done */
if (modth.initialized)
{
return;
}
/* With full OpenMP support (verlet scheme) set the number of threads
* per process / default:
* - 1 if not compiled with OpenMP or
* - OMP_NUM_THREADS if the env. var is set, or
* - omp_nthreads_req = #of threads requested by the user on the mdrun
* command line, otherwise
* - take the max number of available threads and distribute them
* on the processes/tMPI threads.
* ~ The GMX_*_NUM_THREADS env var overrides the number of threads of
* the respective module and it has to be used in conjunction with
* OMP_NUM_THREADS.
*
* With the group scheme OpenMP multithreading is only supported in PME,
* for all other modules nthreads is set to 1.
* The number of PME threads is equal to:
* - 1 if not compiled with OpenMP or
* - GMX_PME_NUM_THREADS if defined, otherwise
* - OMP_NUM_THREADS if defined, otherwise
* - 1
*/
nth = 1;
if ((env = getenv("OMP_NUM_THREADS")) != NULL)
{
if (!bOMP && (strncmp(env, "1", 1) != 0))
{
gmx_warning("OMP_NUM_THREADS is set, but %s was compiled without OpenMP support!",
ShortProgram());
}
else
{
nth = gmx_omp_get_max_threads();
}
}
else if (omp_nthreads_req > 0)
{
nth = omp_nthreads_req;
}
else if (bFullOmpSupport && bOMP)
{
/* max available threads per node */
nth = nthreads_hw_avail;
/* divide the threads among the MPI processes/tMPI threads */
if (nth >= nppn)
{
nth /= nppn;
}
else
{
nth = 1;
}
}
/* now we have the global values, set them:
* - 1 if not compiled with OpenMP and for the group scheme
* - nth for the verlet scheme when compiled with OpenMP
*/
if (bFullOmpSupport && bOMP)
{
modth.gnth = nth;
}
else
{
modth.gnth = 1;
}
if (bSepPME)
{
if (omp_nthreads_pme_req > 0)
{
modth.gnth_pme = omp_nthreads_pme_req;
}
else
{
modth.gnth_pme = nth;
}
}
else
{
modth.gnth_pme = 0;
}
/* now set the per-module values */
modth.nth[emntDefault] = modth.gnth;
pick_module_nthreads(fplog, emntDomdec, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntPairsearch, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntNonbonded, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntBonded, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntPME, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntUpdate, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntVSITE, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntLINCS, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntSETTLE, SIMMASTER(cr), bFullOmpSupport, bSepPME);
/* set the number of threads globally */
if (bOMP)
{
#ifndef GMX_THREAD_MPI
if (bThisNodePMEOnly)
{
gmx_omp_set_num_threads(modth.gnth_pme);
}
else
#endif /* GMX_THREAD_MPI */
{
if (bFullOmpSupport)
{
gmx_omp_set_num_threads(nth);
}
else
{
gmx_omp_set_num_threads(1);
}
}
}
modth.initialized = TRUE;
}
#ifdef GMX_THREAD_MPI
/* Non-master threads have to wait for the detection to be done. */
if (PAR(cr))
{
MPI_Barrier(cr->mpi_comm_mysim);
}
#endif
/* inform the user about the settings */
if (bOMP)
{
#ifdef GMX_THREAD_MPI
const char *mpi_str = "per tMPI thread";
#else
const char *mpi_str = "per MPI process";
#endif
/* for group scheme we print PME threads info only */
if (bFullOmpSupport)
{
md_print_info(cr, fplog, "Using %d OpenMP thread%s %s\n",
modth.gnth, modth.gnth > 1 ? "s" : "",
cr->nnodes > 1 ? mpi_str : "");
}
if (bSepPME && modth.gnth_pme != modth.gnth)
{
md_print_info(cr, fplog, "Using %d OpenMP thread%s %s for PME\n",
modth.gnth_pme, modth.gnth_pme > 1 ? "s" : "",
cr->nnodes > 1 ? mpi_str : "");
}
}
/* detect and warn about oversubscription
* TODO: enable this for separate PME nodes as well! */
if (!bSepPME && cr->rank_pp_intranode == 0)
{
char sbuf[STRLEN], sbuf1[STRLEN], sbuf2[STRLEN];
if (modth.gnth*nppn > nthreads_hw_avail)
{
sprintf(sbuf, "threads");
sbuf1[0] = '\0';
sprintf(sbuf2, "O");
#ifdef GMX_MPI
if (modth.gnth == 1)
{
#ifdef GMX_THREAD_MPI
sprintf(sbuf, "thread-MPI threads");
#else
sprintf(sbuf, "MPI processes");
sprintf(sbuf1, " per node");
sprintf(sbuf2, "On node %d: o", cr->sim_nodeid);
#endif
}
#endif
md_print_warn(cr, fplog,
"WARNING: %sversubscribing the available %d logical CPU cores%s with %d %s.\n"
" This will cause considerable performance loss!",
sbuf2, nthreads_hw_avail, sbuf1, nppn*modth.gnth, sbuf);
}
}
}
