void init_multisystem(t_commrec *cr, int nsim, char **multidirs,
int nfile, const t_filenm fnm[], gmx_bool bParFn)
{
gmx_multisim_t *ms;
int nnodes, nnodpersim, sim, i, ftp;
char buf[256];
#ifdef GMX_MPI
MPI_Group mpi_group_world;
int *rank;
#endif
#ifndef GMX_MPI
if (nsim > 1)
{
gmx_fatal(FARGS, "This binary is compiled without MPI support, can not do multiple simulations.");
}
#endif
nnodes = cr->nnodes;
if (nnodes % nsim != 0)
{
gmx_fatal(FARGS, "The number of ranks (%d) is not a multiple of the number of simulations (%d)", nnodes, nsim);
}
nnodpersim = nnodes/nsim;
sim = cr->nodeid/nnodpersim;
if (debug)
{
fprintf(debug, "We have %d simulations, %d ranks per simulation, local simulation is %d\n", nsim, nnodpersim, sim);
}
snew(ms, 1);
cr->ms = ms;
ms->nsim = nsim;
ms->sim = sim;
#ifdef GMX_MPI
/* Create a communicator for the master nodes */
snew(rank, ms->nsim);
for (i = 0; i < ms->nsim; i++)
{
rank[i] = i*nnodpersim;
}
MPI_Comm_group(MPI_COMM_WORLD, &mpi_group_world);
MPI_Group_incl(mpi_group_world, nsim, rank, &ms->mpi_group_masters);
sfree(rank);
MPI_Comm_create(MPI_COMM_WORLD, ms->mpi_group_masters,
&ms->mpi_comm_masters);
#if !defined(MPI_IN_PLACE_EXISTS)
/* initialize the MPI_IN_PLACE replacement buffers */
snew(ms->mpb, 1);
ms->mpb->ibuf = NULL;
ms->mpb->libuf = NULL;
ms->mpb->fbuf = NULL;
ms->mpb->dbuf = NULL;
ms->mpb->ibuf_alloc = 0;
ms->mpb->libuf_alloc = 0;
ms->mpb->fbuf_alloc = 0;
ms->mpb->dbuf_alloc = 0;
#endif
#endif
/* Reduce the intra-simulation communication */
cr->sim_nodeid = cr->nodeid % nnodpersim;
cr->nnodes = nnodpersim;
#ifdef GMX_MPI
MPI_Comm_split(MPI_COMM_WORLD, sim, cr->sim_nodeid, &cr->mpi_comm_mysim);
cr->mpi_comm_mygroup = cr->mpi_comm_mysim;
cr->nodeid = cr->sim_nodeid;
#endif
if (debug)
{
fprintf(debug, "This is simulation %d", cr->ms->sim);
if (PAR(cr))
{
fprintf(debug, ", local number of ranks %d, local rank ID %d",
cr->nnodes, cr->sim_nodeid);
}
fprintf(debug, "\n\n");
}
if (multidirs)
{
if (debug)
{
fprintf(debug, "Changing to directory %s\n", multidirs[cr->ms->sim]);
}
gmx_chdir(multidirs[cr->ms->sim]);
}
else if (bParFn)
{
/* Patch output and tpx, cpt and rerun input file names */
for (i = 0; (i < nfile); i++)
{
/* Because of possible multiple extensions per type we must look
* at the actual file name
*/
if (is_output(&fnm[i]) ||
fnm[i].ftp == efTPX || fnm[i].ftp == efCPT ||
strcmp(fnm[i].opt, "-rerun") == 0)
{
ftp = fn2ftp(fnm[i].fns[0]);
par_fn(fnm[i].fns[0], ftp, cr, TRUE, FALSE, buf, 255);
sfree(fnm[i].fns[0]);
fnm[i].fns[0] = gmx_strdup(buf);
}
}
}
}
/* Open the file to be processed. The handle variable holds internal
info for the cpp emulator. Return integer status */
int cpp_open_file(const char *filenm, gmx_cpp_t *handle, char **cppopts)
{
gmx_cpp_t cpp;
char *buf, *pdum;
char *ptr, *ptr2;
int i;
unsigned int i1;
/* First process options, they might be necessary for opening files
(especially include statements). */
i = 0;
if (cppopts)
{
while (cppopts[i])
{
if (strstr(cppopts[i], "-I") == cppopts[i])
{
add_include(cppopts[i]+2);
}
if (strstr(cppopts[i], "-D") == cppopts[i])
{
/* If the option contains a =, split it into name and value. */
ptr = strchr(cppopts[i], '=');
if (ptr)
{
buf = gmx_strndup(cppopts[i] + 2, ptr - cppopts[i] - 2);
add_define(buf, ptr + 1);
sfree(buf);
}
else
{
add_define(cppopts[i] + 2, NULL);
}
}
i++;
}
}
if (debug)
{
fprintf(debug, "GMXCPP: added %d command line arguments\n", i);
}
snew(cpp, 1);
*handle = cpp;
cpp->fn = NULL;
/* Find the file. First check whether it is in the current directory. */
if (gmx_fexist(filenm))
{
cpp->fn = gmx_strdup(filenm);
}
else
{
/* If not, check all the paths given with -I. */
for (i = 0; i < nincl; ++i)
{
snew(buf, strlen(incl[i]) + strlen(filenm) + 2);
sprintf(buf, "%s/%s", incl[i], filenm);
if (gmx_fexist(buf))
{
cpp->fn = buf;
break;
}
sfree(buf);
}
/* If still not found, check the Gromacs library search path. */
if (!cpp->fn)
{
cpp->fn = low_gmxlibfn(filenm, FALSE, FALSE);
}
}
if (!cpp->fn)
{
gmx_fatal(FARGS, "Topology include file \"%s\" not found", filenm);
}
if (NULL != debug)
{
fprintf(debug, "GMXCPP: cpp file open %s\n", cpp->fn);
}
/* If the file name has a path component, we need to change to that
* directory. Note that we - just as C - always use UNIX path separators
* internally in include file names.
*/
ptr = strrchr(cpp->fn, '/');
ptr2 = strrchr(cpp->fn, DIR_SEPARATOR);
if (ptr == NULL || (ptr2 != NULL && ptr2 > ptr))
{
ptr = ptr2;
}
if (ptr == NULL)
{
cpp->path = NULL;
cpp->cwd = NULL;
}
else
{
cpp->path = cpp->fn;
*ptr = '\0';
cpp->fn = gmx_strdup(ptr+1);
snew(cpp->cwd, STRLEN);
gmx_getcwd(cpp->cwd, STRLEN);
if (NULL != debug)
{
fprintf(debug, "GMXCPP: cwd %s\n", cpp->cwd);
}
gmx_chdir(cpp->path);
if (NULL != debug)
{
fprintf(debug, "GMXCPP: chdir to %s\n", cpp->path);
}
}
cpp->line_len = 0;
cpp->line = NULL;
cpp->line_nr = 0;
cpp->nifdef = 0;
cpp->ifdefs = NULL;
cpp->child = NULL;
cpp->parent = NULL;
if (cpp->fp == NULL)
{
if (NULL != debug)
{
fprintf(debug, "GMXCPP: opening file %s\n", cpp->fn);
}
cpp->fp = fopen(cpp->fn, "r");
}
if (cpp->fp == NULL)
{
switch (errno)
{
case EINVAL:
default:
return eCPP_UNKNOWN;
}
}
return eCPP_OK;
}
/* Close the file! Return integer status. */
int cpp_close_file(gmx_cpp_t *handlep)
{
int i;
gmx_cpp_t handle = (gmx_cpp_t)*handlep;
if (!handle)
{
return eCPP_INVALID_HANDLE;
}
if (!handle->fp)
{
return eCPP_FILE_NOT_OPEN;
}
if (debug)
{
fprintf(debug, "GMXCPP: closing file %s\n", handle->fn);
}
fclose(handle->fp);
if (NULL != handle->cwd)
{
if (NULL != debug)
{
fprintf(debug, "GMXCPP: chdir to %s\n", handle->cwd);
}
gmx_chdir(handle->cwd);
}
if (0)
{
switch (errno)
{
case 0:
break;
case ENOENT:
return eCPP_FILE_NOT_FOUND;
case EBADF:
return eCPP_FILE_NOT_OPEN;
case EINTR:
return eCPP_INTERRUPT;
default:
if (debug)
{
fprintf(debug, "Strange stuff closing file, errno = %d", errno);
}
return eCPP_UNKNOWN;
}
}
handle->fp = NULL;
handle->line_nr = 0;
if (NULL != handle->fn)
{
sfree(handle->fn);
handle->fn = NULL;
}
if (NULL != handle->line)
{
sfree(handle->line);
handle->line = NULL;
}
if (NULL != handle->ifdefs)
{
sfree(handle->ifdefs);
}
handle->nifdef = 0;
if (NULL != handle->path)
{
sfree(handle->path);
}
if (NULL != handle->cwd)
{
sfree(handle->cwd);
}
return eCPP_OK;
}
gmx_multisim_t *init_multisystem(MPI_Comm comm,
gmx::ArrayRef<const std::string> multidirs)
{
gmx_multisim_t *ms;
#if GMX_MPI
MPI_Group mpi_group_world;
int *rank;
#endif
if (multidirs.empty())
{
return nullptr;
}
if (!GMX_LIB_MPI && !multidirs.empty())
{
gmx_fatal(FARGS, "mdrun -multidir is only supported when GROMACS has been "
"configured with a proper external MPI library.");
}
if (multidirs.size() == 1)
{
/* NOTE: It would be nice if this special case worked, but this requires checks/tests. */
gmx_fatal(FARGS, "To run mdrun in multiple simulation mode, more then one "
"actual simulation is required. The single simulation case is not supported.");
}
#if GMX_MPI
int numRanks;
MPI_Comm_size(comm, &numRanks);
if (numRanks % multidirs.size() != 0)
{
gmx_fatal(FARGS, "The number of ranks (%d) is not a multiple of the number of simulations (%td)", numRanks, multidirs.size());
}
int numRanksPerSim = numRanks/multidirs.size();
int rankWithinComm;
MPI_Comm_rank(comm, &rankWithinComm);
if (debug)
{
fprintf(debug, "We have %td simulations, %d ranks per simulation, local simulation is %d\n", multidirs.size(), numRanksPerSim, rankWithinComm/numRanksPerSim);
}
ms = new gmx_multisim_t;
ms->nsim = multidirs.size();
ms->sim = rankWithinComm/numRanksPerSim;
/* Create a communicator for the master nodes */
snew(rank, ms->nsim);
for (int i = 0; i < ms->nsim; i++)
{
rank[i] = i*numRanksPerSim;
}
MPI_Comm_group(comm, &mpi_group_world);
MPI_Group_incl(mpi_group_world, ms->nsim, rank, &ms->mpi_group_masters);
sfree(rank);
MPI_Comm_create(MPI_COMM_WORLD, ms->mpi_group_masters,
&ms->mpi_comm_masters);
#if !MPI_IN_PLACE_EXISTS
/* initialize the MPI_IN_PLACE replacement buffers */
snew(ms->mpb, 1);
ms->mpb->ibuf = NULL;
ms->mpb->libuf = NULL;
ms->mpb->fbuf = NULL;
ms->mpb->dbuf = NULL;
ms->mpb->ibuf_alloc = 0;
ms->mpb->libuf_alloc = 0;
ms->mpb->fbuf_alloc = 0;
ms->mpb->dbuf_alloc = 0;
#endif
// TODO This should throw upon error
gmx_chdir(multidirs[ms->sim].c_str());
#else
GMX_UNUSED_VALUE(comm);
ms = nullptr;
#endif
return ms;
}
