/*! \brief Helper function for parsing various input about the number of OpenMP threads to use in various modules and deciding what to do about it. */ static void manage_number_of_openmp_threads(const gmx::MDLogger &mdlog, const t_commrec *cr, bool bOMP, int nthreads_hw_avail, int omp_nthreads_req, int omp_nthreads_pme_req, gmx_bool gmx_unused bThisNodePMEOnly, gmx_bool bFullOmpSupport, int numRanksOnThisNode, gmx_bool bSepPME) { int nth; char *env; #if GMX_THREAD_MPI /* modth is shared among tMPI threads, so for thread safety, the * detection is done on the master only. It is not thread-safe * with multiple simulations, but that's anyway not supported by * tMPI. */ if (!SIMMASTER(cr)) { return; } #else GMX_UNUSED_VALUE(cr); #endif if (modth.initialized) { /* Just return if the initialization has already been done. This could only happen if gmx_omp_nthreads_init() has already been called. */ return; } /* With full OpenMP support (verlet scheme) set the number of threads * per process / default: * - 1 if not compiled with OpenMP or * - OMP_NUM_THREADS if the env. var is set, or * - omp_nthreads_req = #of threads requested by the user on the mdrun * command line, otherwise * - take the max number of available threads and distribute them * on the processes/tMPI threads. * ~ The GMX_*_NUM_THREADS env var overrides the number of threads of * the respective module and it has to be used in conjunction with * OMP_NUM_THREADS. * * With the group scheme OpenMP multithreading is only supported in PME, * for all other modules nthreads is set to 1. * The number of PME threads is equal to: * - 1 if not compiled with OpenMP or * - GMX_PME_NUM_THREADS if defined, otherwise * - OMP_NUM_THREADS if defined, otherwise * - 1 */ nth = 1; if ((env = getenv("OMP_NUM_THREADS")) != nullptr) { if (!bOMP && (std::strncmp(env, "1", 1) != 0)) { gmx_warning("OMP_NUM_THREADS is set, but %s was compiled without OpenMP support!", gmx::getProgramContext().displayName()); } else { nth = gmx_omp_get_max_threads(); } } else if (omp_nthreads_req > 0) { nth = omp_nthreads_req; } else if (bFullOmpSupport && bOMP) { /* max available threads per node */ nth = nthreads_hw_avail; /* divide the threads among the MPI ranks */ if (nth >= numRanksOnThisNode) { nth /= numRanksOnThisNode; } else { nth = 1; } } /* now we have the global values, set them: * - 1 if not compiled with OpenMP and for the group scheme * - nth for the verlet scheme when compiled with OpenMP */ if (bFullOmpSupport && bOMP) { modth.gnth = nth; } else { modth.gnth = 1; } if (bSepPME) { if (omp_nthreads_pme_req > 0) { modth.gnth_pme = omp_nthreads_pme_req; } else { modth.gnth_pme = nth; } } else { modth.gnth_pme = 0; } /* now set the per-module values */ modth.nth[emntDefault] = modth.gnth; pick_module_nthreads(mdlog, emntDomdec, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntPairsearch, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntNonbonded, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntBonded, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntPME, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntUpdate, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntVSITE, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntLINCS, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntSETTLE, bFullOmpSupport, bSepPME); /* set the number of threads globally */ if (bOMP) { #if !GMX_THREAD_MPI if (bThisNodePMEOnly) { gmx_omp_set_num_threads(modth.gnth_pme); } else #endif /* GMX_THREAD_MPI */ { if (bFullOmpSupport) { gmx_omp_set_num_threads(nth); } else { gmx_omp_set_num_threads(1); } } } modth.initialized = TRUE; }
void gmx_omp_nthreads_init(FILE *fplog, t_commrec *cr, int nthreads_hw_avail, int omp_nthreads_req, int omp_nthreads_pme_req, gmx_bool gmx_unused bThisNodePMEOnly, gmx_bool bFullOmpSupport) { int nth, nth_pmeonly, gmx_maxth, nppn; char *env; gmx_bool bSepPME, bOMP; #ifdef GMX_OPENMP bOMP = TRUE; #else bOMP = FALSE; #endif /* GMX_OPENMP */ /* number of MPI processes/threads per physical node */ nppn = cr->nrank_intranode; bSepPME = ( (cr->duty & DUTY_PP) && !(cr->duty & DUTY_PME)) || (!(cr->duty & DUTY_PP) && (cr->duty & DUTY_PME)); #ifdef GMX_THREAD_MPI /* modth is shared among tMPI threads, so for thread safety do the * detection is done on the master only. It is not thread-safe with * multiple simulations, but that's anyway not supported by tMPI. */ if (SIMMASTER(cr)) #endif { /* just return if the initialization has already been done */ if (modth.initialized) { return; } /* With full OpenMP support (verlet scheme) set the number of threads * per process / default: * - 1 if not compiled with OpenMP or * - OMP_NUM_THREADS if the env. var is set, or * - omp_nthreads_req = #of threads requested by the user on the mdrun * command line, otherwise * - take the max number of available threads and distribute them * on the processes/tMPI threads. * ~ The GMX_*_NUM_THREADS env var overrides the number of threads of * the respective module and it has to be used in conjunction with * OMP_NUM_THREADS. * * With the group scheme OpenMP multithreading is only supported in PME, * for all other modules nthreads is set to 1. * The number of PME threads is equal to: * - 1 if not compiled with OpenMP or * - GMX_PME_NUM_THREADS if defined, otherwise * - OMP_NUM_THREADS if defined, otherwise * - 1 */ nth = 1; if ((env = getenv("OMP_NUM_THREADS")) != NULL) { if (!bOMP && (strncmp(env, "1", 1) != 0)) { gmx_warning("OMP_NUM_THREADS is set, but %s was compiled without OpenMP support!", ShortProgram()); } else { nth = gmx_omp_get_max_threads(); } } else if (omp_nthreads_req > 0) { nth = omp_nthreads_req; } else if (bFullOmpSupport && bOMP) { /* max available threads per node */ nth = nthreads_hw_avail; /* divide the threads among the MPI processes/tMPI threads */ if (nth >= nppn) { nth /= nppn; } else { nth = 1; } } /* now we have the global values, set them: * - 1 if not compiled with OpenMP and for the group scheme * - nth for the verlet scheme when compiled with OpenMP */ if (bFullOmpSupport && bOMP) { modth.gnth = nth; } else { modth.gnth = 1; } if (bSepPME) { if (omp_nthreads_pme_req > 0) { modth.gnth_pme = omp_nthreads_pme_req; } else { modth.gnth_pme = nth; } } else { modth.gnth_pme = 0; } /* now set the per-module values */ modth.nth[emntDefault] = modth.gnth; pick_module_nthreads(fplog, emntDomdec, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntPairsearch, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntNonbonded, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntBonded, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntPME, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntUpdate, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntVSITE, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntLINCS, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntSETTLE, SIMMASTER(cr), bFullOmpSupport, bSepPME); /* set the number of threads globally */ if (bOMP) { #ifndef GMX_THREAD_MPI if (bThisNodePMEOnly) { gmx_omp_set_num_threads(modth.gnth_pme); } else #endif /* GMX_THREAD_MPI */ { if (bFullOmpSupport) { gmx_omp_set_num_threads(nth); } else { gmx_omp_set_num_threads(1); } } } modth.initialized = TRUE; } #ifdef GMX_THREAD_MPI /* Non-master threads have to wait for the detection to be done. */ if (PAR(cr)) { MPI_Barrier(cr->mpi_comm_mysim); } #endif /* inform the user about the settings */ if (bOMP) { #ifdef GMX_THREAD_MPI const char *mpi_str = "per tMPI thread"; #else const char *mpi_str = "per MPI process"; #endif /* for group scheme we print PME threads info only */ if (bFullOmpSupport) { md_print_info(cr, fplog, "Using %d OpenMP thread%s %s\n", modth.gnth, modth.gnth > 1 ? "s" : "", cr->nnodes > 1 ? mpi_str : ""); } if (bSepPME && modth.gnth_pme != modth.gnth) { md_print_info(cr, fplog, "Using %d OpenMP thread%s %s for PME\n", modth.gnth_pme, modth.gnth_pme > 1 ? "s" : "", cr->nnodes > 1 ? mpi_str : ""); } } /* detect and warn about oversubscription * TODO: enable this for separate PME nodes as well! */ if (!bSepPME && cr->rank_pp_intranode == 0) { char sbuf[STRLEN], sbuf1[STRLEN], sbuf2[STRLEN]; if (modth.gnth*nppn > nthreads_hw_avail) { sprintf(sbuf, "threads"); sbuf1[0] = '\0'; sprintf(sbuf2, "O"); #ifdef GMX_MPI if (modth.gnth == 1) { #ifdef GMX_THREAD_MPI sprintf(sbuf, "thread-MPI threads"); #else sprintf(sbuf, "MPI processes"); sprintf(sbuf1, " per node"); sprintf(sbuf2, "On node %d: o", cr->sim_nodeid); #endif } #endif md_print_warn(cr, fplog, "WARNING: %sversubscribing the available %d logical CPU cores%s with %d %s.\n" " This will cause considerable performance loss!", sbuf2, nthreads_hw_avail, sbuf1, nppn*modth.gnth, sbuf); } } }