/*! \brief Helper function for parsing various input about the number of OpenMP threads to use in various modules and deciding what to do about it. */ static void manage_number_of_openmp_threads(const gmx::MDLogger &mdlog, const t_commrec *cr, bool bOMP, int nthreads_hw_avail, int omp_nthreads_req, int omp_nthreads_pme_req, gmx_bool gmx_unused bThisNodePMEOnly, gmx_bool bFullOmpSupport, int numRanksOnThisNode, gmx_bool bSepPME) { int nth; char *env; #if GMX_THREAD_MPI /* modth is shared among tMPI threads, so for thread safety, the * detection is done on the master only. It is not thread-safe * with multiple simulations, but that's anyway not supported by * tMPI. */ if (!SIMMASTER(cr)) { return; } #else GMX_UNUSED_VALUE(cr); #endif if (modth.initialized) { /* Just return if the initialization has already been done. This could only happen if gmx_omp_nthreads_init() has already been called. */ return; } /* With full OpenMP support (verlet scheme) set the number of threads * per process / default: * - 1 if not compiled with OpenMP or * - OMP_NUM_THREADS if the env. var is set, or * - omp_nthreads_req = #of threads requested by the user on the mdrun * command line, otherwise * - take the max number of available threads and distribute them * on the processes/tMPI threads. * ~ The GMX_*_NUM_THREADS env var overrides the number of threads of * the respective module and it has to be used in conjunction with * OMP_NUM_THREADS. * * With the group scheme OpenMP multithreading is only supported in PME, * for all other modules nthreads is set to 1. * The number of PME threads is equal to: * - 1 if not compiled with OpenMP or * - GMX_PME_NUM_THREADS if defined, otherwise * - OMP_NUM_THREADS if defined, otherwise * - 1 */ nth = 1; if ((env = getenv("OMP_NUM_THREADS")) != nullptr) { if (!bOMP && (std::strncmp(env, "1", 1) != 0)) { gmx_warning("OMP_NUM_THREADS is set, but %s was compiled without OpenMP support!", gmx::getProgramContext().displayName()); } else { nth = gmx_omp_get_max_threads(); } } else if (omp_nthreads_req > 0) { nth = omp_nthreads_req; } else if (bFullOmpSupport && bOMP) { /* max available threads per node */ nth = nthreads_hw_avail; /* divide the threads among the MPI ranks */ if (nth >= numRanksOnThisNode) { nth /= numRanksOnThisNode; } else { nth = 1; } } /* now we have the global values, set them: * - 1 if not compiled with OpenMP and for the group scheme * - nth for the verlet scheme when compiled with OpenMP */ if (bFullOmpSupport && bOMP) { modth.gnth = nth; } else { modth.gnth = 1; } if (bSepPME) { if (omp_nthreads_pme_req > 0) { modth.gnth_pme = omp_nthreads_pme_req; } else { modth.gnth_pme = nth; } } else { modth.gnth_pme = 0; } /* now set the per-module values */ modth.nth[emntDefault] = modth.gnth; pick_module_nthreads(mdlog, emntDomdec, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntPairsearch, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntNonbonded, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntBonded, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntPME, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntUpdate, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntVSITE, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntLINCS, bFullOmpSupport, bSepPME); pick_module_nthreads(mdlog, emntSETTLE, bFullOmpSupport, bSepPME); /* set the number of threads globally */ if (bOMP) { #if !GMX_THREAD_MPI if (bThisNodePMEOnly) { gmx_omp_set_num_threads(modth.gnth_pme); } else #endif /* GMX_THREAD_MPI */ { if (bFullOmpSupport) { gmx_omp_set_num_threads(nth); } else { gmx_omp_set_num_threads(1); } } } modth.initialized = TRUE; }
int gmx_sans(int argc, char *argv[]) { const char *desc[] = { "[THISMODULE] computes SANS spectra using Debye formula.", "It currently uses topology file (since it need to assigne element for each atom).", "[PAR]", "Parameters:[PAR]" "[TT]-pr[tt] Computes normalized g(r) function averaged over trajectory[PAR]", "[TT]-prframe[tt] Computes normalized g(r) function for each frame[PAR]", "[TT]-sq[tt] Computes SANS intensity curve averaged over trajectory[PAR]", "[TT]-sqframe[tt] Computes SANS intensity curve for each frame[PAR]", "[TT]-startq[tt] Starting q value in nm[PAR]", "[TT]-endq[tt] Ending q value in nm[PAR]", "[TT]-qstep[tt] Stepping in q space[PAR]", "Note: When using Debye direct method computational cost increases as", "1/2 * N * (N - 1) where N is atom number in group of interest.", "[PAR]", "WARNING: If sq or pr specified this tool can produce large number of files! Up to two times larger than number of frames!" }; static gmx_bool bPBC = TRUE; static gmx_bool bNORM = FALSE; static real binwidth = 0.2, grid = 0.05; /* bins shouldnt be smaller then smallest bond (~0.1nm) length */ static real start_q = 0.0, end_q = 2.0, q_step = 0.01; static real mcover = -1; static unsigned int seed = 0; static int nthreads = -1; static const char *emode[] = { NULL, "direct", "mc", NULL }; static const char *emethod[] = { NULL, "debye", "fft", NULL }; gmx_neutron_atomic_structurefactors_t *gnsf; gmx_sans_t *gsans; #define NPA asize(pa) t_pargs pa[] = { { "-bin", FALSE, etREAL, {&binwidth}, "[HIDDEN]Binwidth (nm)" }, { "-mode", FALSE, etENUM, {emode}, "Mode for sans spectra calculation" }, { "-mcover", FALSE, etREAL, {&mcover}, "Monte-Carlo coverage should be -1(default) or (0,1]"}, { "-method", FALSE, etENUM, {emethod}, "[HIDDEN]Method for sans spectra calculation" }, { "-pbc", FALSE, etBOOL, {&bPBC}, "Use periodic boundary conditions for computing distances" }, { "-grid", FALSE, etREAL, {&grid}, "[HIDDEN]Grid spacing (in nm) for FFTs" }, {"-startq", FALSE, etREAL, {&start_q}, "Starting q (1/nm) "}, {"-endq", FALSE, etREAL, {&end_q}, "Ending q (1/nm)"}, { "-qstep", FALSE, etREAL, {&q_step}, "Stepping in q (1/nm)"}, { "-seed", FALSE, etINT, {&seed}, "Random seed for Monte-Carlo"}, #ifdef GMX_OPENMP { "-nt", FALSE, etINT, {&nthreads}, "Number of threads to start"}, #endif }; FILE *fp; const char *fnTPX, *fnNDX, *fnTRX, *fnDAT = NULL; t_trxstatus *status; t_topology *top = NULL; t_atom *atom = NULL; gmx_rmpbc_t gpbc = NULL; gmx_bool bTPX; gmx_bool bFFT = FALSE, bDEBYE = FALSE; gmx_bool bMC = FALSE; int ePBC = -1; matrix box; char title[STRLEN]; rvec *x; int natoms; real t; char **grpname = NULL; atom_id *index = NULL; int isize; int i, j; char *hdr = NULL; char *suffix = NULL; t_filenm *fnmdup = NULL; gmx_radial_distribution_histogram_t *prframecurrent = NULL, *pr = NULL; gmx_static_structurefactor_t *sqframecurrent = NULL, *sq = NULL; output_env_t oenv; #define NFILE asize(fnm) t_filenm fnm[] = { { efTPX, "-s", NULL, ffREAD }, { efTRX, "-f", NULL, ffREAD }, { efNDX, NULL, NULL, ffOPTRD }, { efDAT, "-d", "nsfactor", ffOPTRD }, { efXVG, "-pr", "pr", ffWRITE }, { efXVG, "-sq", "sq", ffWRITE }, { efXVG, "-prframe", "prframe", ffOPTWR }, { efXVG, "-sqframe", "sqframe", ffOPTWR } }; nthreads = gmx_omp_get_max_threads(); if (!parse_common_args(&argc, argv, PCA_CAN_TIME | PCA_TIME_UNIT | PCA_BE_NICE, NFILE, fnm, asize(pa), pa, asize(desc), desc, 0, NULL, &oenv)) { return 0; } /* check that binwidth not smaller than smallers distance */ check_binwidth(binwidth); check_mcover(mcover); /* setting number of omp threads globaly */ gmx_omp_set_num_threads(nthreads); /* Now try to parse opts for modes */ switch (emethod[0][0]) { case 'd': bDEBYE = TRUE; switch (emode[0][0]) { case 'd': bMC = FALSE; break; case 'm': bMC = TRUE; break; default: break; } break; case 'f': bFFT = TRUE; break; default: break; } if (bDEBYE) { if (bMC) { fprintf(stderr, "Using Monte Carlo Debye method to calculate spectrum\n"); } else { fprintf(stderr, "Using direct Debye method to calculate spectrum\n"); } } else if (bFFT) { gmx_fatal(FARGS, "FFT method not implemented!"); } else { gmx_fatal(FARGS, "Unknown combination for mode and method!"); } /* Try to read files */ fnDAT = ftp2fn(efDAT, NFILE, fnm); fnTPX = ftp2fn(efTPX, NFILE, fnm); fnTRX = ftp2fn(efTRX, NFILE, fnm); gnsf = gmx_neutronstructurefactors_init(fnDAT); fprintf(stderr, "Read %d atom names from %s with neutron scattering parameters\n\n", gnsf->nratoms, fnDAT); snew(top, 1); snew(grpname, 1); snew(index, 1); bTPX = read_tps_conf(fnTPX, title, top, &ePBC, &x, NULL, box, TRUE); printf("\nPlease select group for SANS spectra calculation:\n"); get_index(&(top->atoms), ftp2fn_null(efNDX, NFILE, fnm), 1, &isize, &index, grpname); gsans = gmx_sans_init(top, gnsf); /* Prepare reference frame */ if (bPBC) { gpbc = gmx_rmpbc_init(&top->idef, ePBC, top->atoms.nr); gmx_rmpbc(gpbc, top->atoms.nr, box, x); } natoms = read_first_x(oenv, &status, fnTRX, &t, &x, box); if (natoms != top->atoms.nr) { fprintf(stderr, "\nWARNING: number of atoms in tpx (%d) and trajectory (%d) do not match\n", natoms, top->atoms.nr); } do { if (bPBC) { gmx_rmpbc(gpbc, top->atoms.nr, box, x); } /* allocate memory for pr */ if (pr == NULL) { /* in case its first frame to read */ snew(pr, 1); } /* realy calc p(r) */ prframecurrent = calc_radial_distribution_histogram(gsans, x, box, index, isize, binwidth, bMC, bNORM, mcover, seed); /* copy prframecurrent -> pr and summ up pr->gr[i] */ /* allocate and/or resize memory for pr->gr[i] and pr->r[i] */ if (pr->gr == NULL) { /* check if we use pr->gr first time */ snew(pr->gr, prframecurrent->grn); snew(pr->r, prframecurrent->grn); } else { /* resize pr->gr and pr->r if needed to preven overruns */ if (prframecurrent->grn > pr->grn) { srenew(pr->gr, prframecurrent->grn); srenew(pr->r, prframecurrent->grn); } } pr->grn = prframecurrent->grn; pr->binwidth = prframecurrent->binwidth; /* summ up gr and fill r */ for (i = 0; i < prframecurrent->grn; i++) { pr->gr[i] += prframecurrent->gr[i]; pr->r[i] = prframecurrent->r[i]; } /* normalize histo */ normalize_probability(prframecurrent->grn, prframecurrent->gr); /* convert p(r) to sq */ sqframecurrent = convert_histogram_to_intensity_curve(prframecurrent, start_q, end_q, q_step); /* print frame data if needed */ if (opt2fn_null("-prframe", NFILE, fnm)) { snew(hdr, 25); snew(suffix, GMX_PATH_MAX); /* prepare header */ sprintf(hdr, "g(r), t = %f", t); /* prepare output filename */ fnmdup = dup_tfn(NFILE, fnm); sprintf(suffix, "-t%.2f", t); add_suffix_to_output_names(fnmdup, NFILE, suffix); fp = xvgropen(opt2fn_null("-prframe", NFILE, fnmdup), hdr, "Distance (nm)", "Probability", oenv); for (i = 0; i < prframecurrent->grn; i++) { fprintf(fp, "%10.6f%10.6f\n", prframecurrent->r[i], prframecurrent->gr[i]); } done_filenms(NFILE, fnmdup); fclose(fp); sfree(hdr); sfree(suffix); sfree(fnmdup); } if (opt2fn_null("-sqframe", NFILE, fnm)) { snew(hdr, 25); snew(suffix, GMX_PATH_MAX); /* prepare header */ sprintf(hdr, "I(q), t = %f", t); /* prepare output filename */ fnmdup = dup_tfn(NFILE, fnm); sprintf(suffix, "-t%.2f", t); add_suffix_to_output_names(fnmdup, NFILE, suffix); fp = xvgropen(opt2fn_null("-sqframe", NFILE, fnmdup), hdr, "q (nm^-1)", "s(q)/s(0)", oenv); for (i = 0; i < sqframecurrent->qn; i++) { fprintf(fp, "%10.6f%10.6f\n", sqframecurrent->q[i], sqframecurrent->s[i]); } done_filenms(NFILE, fnmdup); fclose(fp); sfree(hdr); sfree(suffix); sfree(fnmdup); } /* free pr structure */ sfree(prframecurrent->gr); sfree(prframecurrent->r); sfree(prframecurrent); /* free sq structure */ sfree(sqframecurrent->q); sfree(sqframecurrent->s); sfree(sqframecurrent); } while (read_next_x(oenv, status, &t, x, box)); close_trj(status); /* normalize histo */ normalize_probability(pr->grn, pr->gr); sq = convert_histogram_to_intensity_curve(pr, start_q, end_q, q_step); /* prepare pr.xvg */ fp = xvgropen(opt2fn_null("-pr", NFILE, fnm), "G(r)", "Distance (nm)", "Probability", oenv); for (i = 0; i < pr->grn; i++) { fprintf(fp, "%10.6f%10.6f\n", pr->r[i], pr->gr[i]); } xvgrclose(fp); /* prepare sq.xvg */ fp = xvgropen(opt2fn_null("-sq", NFILE, fnm), "I(q)", "q (nm^-1)", "s(q)/s(0)", oenv); for (i = 0; i < sq->qn; i++) { fprintf(fp, "%10.6f%10.6f\n", sq->q[i], sq->s[i]); } xvgrclose(fp); /* * Clean up memory */ sfree(pr->gr); sfree(pr->r); sfree(pr); sfree(sq->q); sfree(sq->s); sfree(sq); please_cite(stdout, "Garmay2012"); return 0; }
void gmx_omp_nthreads_init(FILE *fplog, t_commrec *cr, int nthreads_hw_avail, int omp_nthreads_req, int omp_nthreads_pme_req, gmx_bool gmx_unused bThisNodePMEOnly, gmx_bool bFullOmpSupport) { int nth, nth_pmeonly, gmx_maxth, nppn; char *env; gmx_bool bSepPME, bOMP; #ifdef GMX_OPENMP bOMP = TRUE; #else bOMP = FALSE; #endif /* GMX_OPENMP */ /* number of MPI processes/threads per physical node */ nppn = cr->nrank_intranode; bSepPME = ( (cr->duty & DUTY_PP) && !(cr->duty & DUTY_PME)) || (!(cr->duty & DUTY_PP) && (cr->duty & DUTY_PME)); #ifdef GMX_THREAD_MPI /* modth is shared among tMPI threads, so for thread safety do the * detection is done on the master only. It is not thread-safe with * multiple simulations, but that's anyway not supported by tMPI. */ if (SIMMASTER(cr)) #endif { /* just return if the initialization has already been done */ if (modth.initialized) { return; } /* With full OpenMP support (verlet scheme) set the number of threads * per process / default: * - 1 if not compiled with OpenMP or * - OMP_NUM_THREADS if the env. var is set, or * - omp_nthreads_req = #of threads requested by the user on the mdrun * command line, otherwise * - take the max number of available threads and distribute them * on the processes/tMPI threads. * ~ The GMX_*_NUM_THREADS env var overrides the number of threads of * the respective module and it has to be used in conjunction with * OMP_NUM_THREADS. * * With the group scheme OpenMP multithreading is only supported in PME, * for all other modules nthreads is set to 1. * The number of PME threads is equal to: * - 1 if not compiled with OpenMP or * - GMX_PME_NUM_THREADS if defined, otherwise * - OMP_NUM_THREADS if defined, otherwise * - 1 */ nth = 1; if ((env = getenv("OMP_NUM_THREADS")) != NULL) { if (!bOMP && (strncmp(env, "1", 1) != 0)) { gmx_warning("OMP_NUM_THREADS is set, but %s was compiled without OpenMP support!", ShortProgram()); } else { nth = gmx_omp_get_max_threads(); } } else if (omp_nthreads_req > 0) { nth = omp_nthreads_req; } else if (bFullOmpSupport && bOMP) { /* max available threads per node */ nth = nthreads_hw_avail; /* divide the threads among the MPI processes/tMPI threads */ if (nth >= nppn) { nth /= nppn; } else { nth = 1; } } /* now we have the global values, set them: * - 1 if not compiled with OpenMP and for the group scheme * - nth for the verlet scheme when compiled with OpenMP */ if (bFullOmpSupport && bOMP) { modth.gnth = nth; } else { modth.gnth = 1; } if (bSepPME) { if (omp_nthreads_pme_req > 0) { modth.gnth_pme = omp_nthreads_pme_req; } else { modth.gnth_pme = nth; } } else { modth.gnth_pme = 0; } /* now set the per-module values */ modth.nth[emntDefault] = modth.gnth; pick_module_nthreads(fplog, emntDomdec, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntPairsearch, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntNonbonded, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntBonded, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntPME, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntUpdate, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntVSITE, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntLINCS, SIMMASTER(cr), bFullOmpSupport, bSepPME); pick_module_nthreads(fplog, emntSETTLE, SIMMASTER(cr), bFullOmpSupport, bSepPME); /* set the number of threads globally */ if (bOMP) { #ifndef GMX_THREAD_MPI if (bThisNodePMEOnly) { gmx_omp_set_num_threads(modth.gnth_pme); } else #endif /* GMX_THREAD_MPI */ { if (bFullOmpSupport) { gmx_omp_set_num_threads(nth); } else { gmx_omp_set_num_threads(1); } } } modth.initialized = TRUE; } #ifdef GMX_THREAD_MPI /* Non-master threads have to wait for the detection to be done. */ if (PAR(cr)) { MPI_Barrier(cr->mpi_comm_mysim); } #endif /* inform the user about the settings */ if (bOMP) { #ifdef GMX_THREAD_MPI const char *mpi_str = "per tMPI thread"; #else const char *mpi_str = "per MPI process"; #endif /* for group scheme we print PME threads info only */ if (bFullOmpSupport) { md_print_info(cr, fplog, "Using %d OpenMP thread%s %s\n", modth.gnth, modth.gnth > 1 ? "s" : "", cr->nnodes > 1 ? mpi_str : ""); } if (bSepPME && modth.gnth_pme != modth.gnth) { md_print_info(cr, fplog, "Using %d OpenMP thread%s %s for PME\n", modth.gnth_pme, modth.gnth_pme > 1 ? "s" : "", cr->nnodes > 1 ? mpi_str : ""); } } /* detect and warn about oversubscription * TODO: enable this for separate PME nodes as well! */ if (!bSepPME && cr->rank_pp_intranode == 0) { char sbuf[STRLEN], sbuf1[STRLEN], sbuf2[STRLEN]; if (modth.gnth*nppn > nthreads_hw_avail) { sprintf(sbuf, "threads"); sbuf1[0] = '\0'; sprintf(sbuf2, "O"); #ifdef GMX_MPI if (modth.gnth == 1) { #ifdef GMX_THREAD_MPI sprintf(sbuf, "thread-MPI threads"); #else sprintf(sbuf, "MPI processes"); sprintf(sbuf1, " per node"); sprintf(sbuf2, "On node %d: o", cr->sim_nodeid); #endif } #endif md_print_warn(cr, fplog, "WARNING: %sversubscribing the available %d logical CPU cores%s with %d %s.\n" " This will cause considerable performance loss!", sbuf2, nthreads_hw_avail, sbuf1, nppn*modth.gnth, sbuf); } } }