/*! \brief Helper function for parsing various input about the number
of OpenMP threads to use in various modules and deciding what to
do about it. */
static void manage_number_of_openmp_threads(const gmx::MDLogger &mdlog,
const t_commrec *cr,
bool bOMP,
int nthreads_hw_avail,
int omp_nthreads_req,
int omp_nthreads_pme_req,
gmx_bool gmx_unused bThisNodePMEOnly,
gmx_bool bFullOmpSupport,
int numRanksOnThisNode,
gmx_bool bSepPME)
{
int nth;
char *env;
#if GMX_THREAD_MPI
/* modth is shared among tMPI threads, so for thread safety, the
* detection is done on the master only. It is not thread-safe
* with multiple simulations, but that's anyway not supported by
* tMPI. */
if (!SIMMASTER(cr))
{
return;
}
#else
GMX_UNUSED_VALUE(cr);
#endif
if (modth.initialized)
{
/* Just return if the initialization has already been
done. This could only happen if gmx_omp_nthreads_init() has
already been called. */
return;
}
/* With full OpenMP support (verlet scheme) set the number of threads
* per process / default:
* - 1 if not compiled with OpenMP or
* - OMP_NUM_THREADS if the env. var is set, or
* - omp_nthreads_req = #of threads requested by the user on the mdrun
* command line, otherwise
* - take the max number of available threads and distribute them
* on the processes/tMPI threads.
* ~ The GMX_*_NUM_THREADS env var overrides the number of threads of
* the respective module and it has to be used in conjunction with
* OMP_NUM_THREADS.
*
* With the group scheme OpenMP multithreading is only supported in PME,
* for all other modules nthreads is set to 1.
* The number of PME threads is equal to:
* - 1 if not compiled with OpenMP or
* - GMX_PME_NUM_THREADS if defined, otherwise
* - OMP_NUM_THREADS if defined, otherwise
* - 1
*/
nth = 1;
if ((env = getenv("OMP_NUM_THREADS")) != nullptr)
{
if (!bOMP && (std::strncmp(env, "1", 1) != 0))
{
gmx_warning("OMP_NUM_THREADS is set, but %s was compiled without OpenMP support!",
gmx::getProgramContext().displayName());
}
else
{
nth = gmx_omp_get_max_threads();
}
}
else if (omp_nthreads_req > 0)
{
nth = omp_nthreads_req;
}
else if (bFullOmpSupport && bOMP)
{
/* max available threads per node */
nth = nthreads_hw_avail;
/* divide the threads among the MPI ranks */
if (nth >= numRanksOnThisNode)
{
nth /= numRanksOnThisNode;
}
else
{
nth = 1;
}
}
/* now we have the global values, set them:
* - 1 if not compiled with OpenMP and for the group scheme
* - nth for the verlet scheme when compiled with OpenMP
*/
if (bFullOmpSupport && bOMP)
{
modth.gnth = nth;
}
else
{
modth.gnth = 1;
}
if (bSepPME)
{
if (omp_nthreads_pme_req > 0)
{
modth.gnth_pme = omp_nthreads_pme_req;
}
else
{
modth.gnth_pme = nth;
}
}
else
{
modth.gnth_pme = 0;
}
/* now set the per-module values */
modth.nth[emntDefault] = modth.gnth;
pick_module_nthreads(mdlog, emntDomdec, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntPairsearch, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntNonbonded, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntBonded, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntPME, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntUpdate, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntVSITE, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntLINCS, bFullOmpSupport, bSepPME);
pick_module_nthreads(mdlog, emntSETTLE, bFullOmpSupport, bSepPME);
/* set the number of threads globally */
if (bOMP)
{
#if !GMX_THREAD_MPI
if (bThisNodePMEOnly)
{
gmx_omp_set_num_threads(modth.gnth_pme);
}
else
#endif /* GMX_THREAD_MPI */
{
if (bFullOmpSupport)
{
gmx_omp_set_num_threads(nth);
}
else
{
gmx_omp_set_num_threads(1);
}
}
}
modth.initialized = TRUE;
}
void gmx_omp_nthreads_init(FILE *fplog, t_commrec *cr,
int nthreads_hw_avail,
int omp_nthreads_req,
int omp_nthreads_pme_req,
gmx_bool gmx_unused bThisNodePMEOnly,
gmx_bool bFullOmpSupport)
{
int nth, nth_pmeonly, gmx_maxth, nppn;
char *env;
gmx_bool bSepPME, bOMP;
#ifdef GMX_OPENMP
bOMP = TRUE;
#else
bOMP = FALSE;
#endif /* GMX_OPENMP */
/* number of MPI processes/threads per physical node */
nppn = cr->nrank_intranode;
bSepPME = ( (cr->duty & DUTY_PP) && !(cr->duty & DUTY_PME)) ||
(!(cr->duty & DUTY_PP) && (cr->duty & DUTY_PME));
#ifdef GMX_THREAD_MPI
/* modth is shared among tMPI threads, so for thread safety do the
* detection is done on the master only. It is not thread-safe with
* multiple simulations, but that's anyway not supported by tMPI. */
if (SIMMASTER(cr))
#endif
{
/* just return if the initialization has already been done */
if (modth.initialized)
{
return;
}
/* With full OpenMP support (verlet scheme) set the number of threads
* per process / default:
* - 1 if not compiled with OpenMP or
* - OMP_NUM_THREADS if the env. var is set, or
* - omp_nthreads_req = #of threads requested by the user on the mdrun
* command line, otherwise
* - take the max number of available threads and distribute them
* on the processes/tMPI threads.
* ~ The GMX_*_NUM_THREADS env var overrides the number of threads of
* the respective module and it has to be used in conjunction with
* OMP_NUM_THREADS.
*
* With the group scheme OpenMP multithreading is only supported in PME,
* for all other modules nthreads is set to 1.
* The number of PME threads is equal to:
* - 1 if not compiled with OpenMP or
* - GMX_PME_NUM_THREADS if defined, otherwise
* - OMP_NUM_THREADS if defined, otherwise
* - 1
*/
nth = 1;
if ((env = getenv("OMP_NUM_THREADS")) != NULL)
{
if (!bOMP && (strncmp(env, "1", 1) != 0))
{
gmx_warning("OMP_NUM_THREADS is set, but %s was compiled without OpenMP support!",
ShortProgram());
}
else
{
nth = gmx_omp_get_max_threads();
}
}
else if (omp_nthreads_req > 0)
{
nth = omp_nthreads_req;
}
else if (bFullOmpSupport && bOMP)
{
/* max available threads per node */
nth = nthreads_hw_avail;
/* divide the threads among the MPI processes/tMPI threads */
if (nth >= nppn)
{
nth /= nppn;
}
else
{
nth = 1;
}
}
/* now we have the global values, set them:
* - 1 if not compiled with OpenMP and for the group scheme
* - nth for the verlet scheme when compiled with OpenMP
*/
if (bFullOmpSupport && bOMP)
{
modth.gnth = nth;
}
else
{
modth.gnth = 1;
}
if (bSepPME)
{
if (omp_nthreads_pme_req > 0)
{
modth.gnth_pme = omp_nthreads_pme_req;
}
else
{
modth.gnth_pme = nth;
}
}
else
{
modth.gnth_pme = 0;
}
/* now set the per-module values */
modth.nth[emntDefault] = modth.gnth;
pick_module_nthreads(fplog, emntDomdec, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntPairsearch, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntNonbonded, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntBonded, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntPME, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntUpdate, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntVSITE, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntLINCS, SIMMASTER(cr), bFullOmpSupport, bSepPME);
pick_module_nthreads(fplog, emntSETTLE, SIMMASTER(cr), bFullOmpSupport, bSepPME);
/* set the number of threads globally */
if (bOMP)
{
#ifndef GMX_THREAD_MPI
if (bThisNodePMEOnly)
{
gmx_omp_set_num_threads(modth.gnth_pme);
}
else
#endif /* GMX_THREAD_MPI */
{
if (bFullOmpSupport)
{
gmx_omp_set_num_threads(nth);
}
else
{
gmx_omp_set_num_threads(1);
}
}
}
modth.initialized = TRUE;
}
#ifdef GMX_THREAD_MPI
/* Non-master threads have to wait for the detection to be done. */
if (PAR(cr))
{
MPI_Barrier(cr->mpi_comm_mysim);
}
#endif
/* inform the user about the settings */
if (bOMP)
{
#ifdef GMX_THREAD_MPI
const char *mpi_str = "per tMPI thread";
#else
const char *mpi_str = "per MPI process";
#endif
/* for group scheme we print PME threads info only */
if (bFullOmpSupport)
{
md_print_info(cr, fplog, "Using %d OpenMP thread%s %s\n",
modth.gnth, modth.gnth > 1 ? "s" : "",
cr->nnodes > 1 ? mpi_str : "");
}
if (bSepPME && modth.gnth_pme != modth.gnth)
{
md_print_info(cr, fplog, "Using %d OpenMP thread%s %s for PME\n",
modth.gnth_pme, modth.gnth_pme > 1 ? "s" : "",
cr->nnodes > 1 ? mpi_str : "");
}
}
/* detect and warn about oversubscription
* TODO: enable this for separate PME nodes as well! */
if (!bSepPME && cr->rank_pp_intranode == 0)
{
char sbuf[STRLEN], sbuf1[STRLEN], sbuf2[STRLEN];
if (modth.gnth*nppn > nthreads_hw_avail)
{
sprintf(sbuf, "threads");
sbuf1[0] = '\0';
sprintf(sbuf2, "O");
#ifdef GMX_MPI
if (modth.gnth == 1)
{
#ifdef GMX_THREAD_MPI
sprintf(sbuf, "thread-MPI threads");
#else
sprintf(sbuf, "MPI processes");
sprintf(sbuf1, " per node");
sprintf(sbuf2, "On node %d: o", cr->sim_nodeid);
#endif
}
#endif
md_print_warn(cr, fplog,
"WARNING: %sversubscribing the available %d logical CPU cores%s with %d %s.\n"
" This will cause considerable performance loss!",
sbuf2, nthreads_hw_avail, sbuf1, nppn*modth.gnth, sbuf);
}
}
}
