static void buildbox(int nnode,ivec nbox,matrix box)
{
ivec *BB,bxyz;
int i,j,m,n,n3,ny,*fx,*fy,nbb;
n3 = ipow(nnode,3)*6;
snew(BB,n3);
nbb=0;
snew(fx,nnode+1);
snew(fy,nnode+1);
factorize(nnode,fx);
for(i=0; (i<=nnode); i++) {
for(m=1; (m<=fx[i]); m++) {
bxyz[XX] = ipow(i,m);
ny = nnode/bxyz[XX];
factorize(ny,fy);
for(j=0; (j<=ny); j++) {
for(n=1; (n<=fy[j]); n++) {
bxyz[YY] = ipow(j,n);
bxyz[ZZ] = ny/bxyz[YY];
if (bxyz[ZZ] > 0) {
nbb = add_bb(BB,nbb,bxyz);
}
}
}
}
}
/* Sort boxes and remove doubles */
qsort(BB,nbb,sizeof(BB[0]),iv_comp);
j = 0;
for(i=1; (i<nbb); i++) {
if ((BB[i][XX] != BB[j][XX]) ||
(BB[i][YY] != BB[j][YY]) ||
(BB[i][ZZ] != BB[j][ZZ])) {
j++;
copy_ivec(BB[i],BB[j]);
}
}
nbb = ++j;
/* Sort boxes according to weight */
copy_mat(box,BOX);
qsort(BB,nbb,sizeof(BB[0]),w_comp);
for(i=0; (i<nbb); i++) {
fprintf(stderr,"nbox = %2d %2d %2d [ prod %3d ] area = %12.5f (nm^2)\n",
BB[i][XX],BB[i][YY],BB[i][ZZ],
BB[i][XX]*BB[i][YY]*BB[i][ZZ],
box_weight(BB[i],box));
}
copy_ivec(BB[0],nbox);
sfree(BB);
sfree(fy);
sfree(fx);
}
static void assign_factors(gmx_domdec_t *dd,
real limit,real cutoff,
matrix box,gmx_ddbox_t *ddbox,t_inputrec *ir,
float pbcdxr,int npme,
int ndiv,int *div,int *mdiv,ivec ir_try,ivec opt)
{
int x,y,z,i;
float ce;
if (ndiv == 0)
{
ce = comm_cost_est(dd,limit,cutoff,box,ddbox,ir,pbcdxr,npme,ir_try);
if (ce >= 0 && (opt[XX] == 0 ||
ce < comm_cost_est(dd,limit,cutoff,box,ddbox,ir,pbcdxr,
npme,opt)))
{
copy_ivec(ir_try,opt);
}
return;
}
for(x=mdiv[0]; x>=0; x--)
{
for(i=0; i<x; i++)
{
ir_try[XX] *= div[0];
}
for(y=mdiv[0]-x; y>=0; y--)
{
for(i=0; i<y; i++)
{
ir_try[YY] *= div[0];
}
for(i=0; i<mdiv[0]-x-y; i++)
{
ir_try[ZZ] *= div[0];
}
/* recurse */
assign_factors(dd,limit,cutoff,box,ddbox,ir,pbcdxr,npme,
ndiv-1,div+1,mdiv+1,ir_try,opt);
for(i=0; i<mdiv[0]-x-y; i++)
{
ir_try[ZZ] /= div[0];
}
for(i=0; i<y; i++)
{
ir_try[YY] /= div[0];
}
}
for(i=0; i<x; i++)
{
ir_try[XX] /= div[0];
}
}
}
void gmx_pme_send_switch(t_commrec *cr, ivec grid_size, real ewaldcoeff)
{
#ifdef GMX_MPI
gmx_pme_comm_n_box_t cnb;
if (cr->dd->pme_receive_vir_ener)
{
cnb.flags = PP_PME_SWITCH;
copy_ivec(grid_size,cnb.grid_size);
cnb.ewaldcoeff = ewaldcoeff;
/* We send this, uncommon, message blocking to simplify the code */
MPI_Send(&cnb,sizeof(cnb),MPI_BYTE,
cr->dd->pme_nodeid,0,cr->mpi_comm_mysim);
}
#endif
}
void pmegrid_init(pmegrid_t *grid,
int cx, int cy, int cz,
int x0, int y0, int z0,
int x1, int y1, int z1,
gmx_bool set_alignment,
int pme_order,
real *ptr)
{
int nz, gridsize;
grid->ci[XX] = cx;
grid->ci[YY] = cy;
grid->ci[ZZ] = cz;
grid->offset[XX] = x0;
grid->offset[YY] = y0;
grid->offset[ZZ] = z0;
grid->n[XX] = x1 - x0 + pme_order - 1;
grid->n[YY] = y1 - y0 + pme_order - 1;
grid->n[ZZ] = z1 - z0 + pme_order - 1;
copy_ivec(grid->n, grid->s);
nz = grid->s[ZZ];
set_grid_alignment(&nz, pme_order);
if (set_alignment)
{
grid->s[ZZ] = nz;
}
else if (nz != grid->s[ZZ])
{
gmx_incons("pmegrid_init call with an unaligned z size");
}
grid->order = pme_order;
if (ptr == NULL)
{
gridsize = grid->s[XX]*grid->s[YY]*grid->s[ZZ];
set_gridsize_alignment(&gridsize, pme_order);
snew_aligned(grid->grid, gridsize, SIMD4_ALIGNMENT);
}
else
{
grid->grid = ptr;
}
}
void gmx_pme_send_switchgrid(t_commrec gmx_unused *cr,
ivec gmx_unused grid_size,
real gmx_unused ewaldcoeff_q,
real gmx_unused ewaldcoeff_lj)
{
#ifdef GMX_MPI
gmx_pme_comm_n_box_t cnb;
/* Only let one PP node signal each PME node */
if (cr->dd->pme_receive_vir_ener)
{
cnb.flags = PP_PME_SWITCHGRID;
copy_ivec(grid_size, cnb.grid_size);
cnb.ewaldcoeff_q = ewaldcoeff_q;
cnb.ewaldcoeff_lj = ewaldcoeff_lj;
/* We send this, uncommon, message blocking to simplify the code */
MPI_Send(&cnb, sizeof(cnb), MPI_BYTE,
cr->dd->pme_nodeid, eCommType_CNB, cr->mpi_comm_mysim);
}
#endif
}
static int add_bb(ivec BB[],int n,ivec b)
{
#define SWPX(vv,xx,yy) { int tmp; tmp=vv[xx]; vv[xx] = vv[yy]; vv[yy] = tmp; }
copy_ivec(b,BB[n++]); /* x y z */
SWPX(b,XX,YY);
copy_ivec(b,BB[n++]); /* y x z */
SWPX(b,XX,ZZ);
copy_ivec(b,BB[n++]); /* z x y */
SWPX(b,XX,YY);
copy_ivec(b,BB[n++]); /* x z y */
SWPX(b,XX,ZZ);
copy_ivec(b,BB[n++]); /* y z x */
SWPX(b,XX,YY);
copy_ivec(b,BB[n++]); /* z y x */
SWPX(b,XX,ZZ); /* Back to normal */
#undef SWPX
return n;
}
void pmegrids_init(pmegrids_t *grids,
int nx, int ny, int nz, int nz_base,
int pme_order,
gmx_bool bUseThreads,
int nthread,
int overlap_x,
int overlap_y)
{
ivec n, n_base;
int t, x, y, z, d, i, tfac;
int max_comm_lines = -1;
n[XX] = nx - (pme_order - 1);
n[YY] = ny - (pme_order - 1);
n[ZZ] = nz - (pme_order - 1);
copy_ivec(n, n_base);
n_base[ZZ] = nz_base;
pmegrid_init(&grids->grid, 0, 0, 0, 0, 0, 0, n[XX], n[YY], n[ZZ], FALSE, pme_order,
NULL);
grids->nthread = nthread;
make_subgrid_division(n_base, pme_order-1, grids->nthread, grids->nc);
if (bUseThreads)
{
ivec nst;
int gridsize;
for (d = 0; d < DIM; d++)
{
nst[d] = div_round_up(n[d], grids->nc[d]) + pme_order - 1;
}
set_grid_alignment(&nst[ZZ], pme_order);
if (debug)
{
fprintf(debug, "pmegrid thread local division: %d x %d x %d\n",
grids->nc[XX], grids->nc[YY], grids->nc[ZZ]);
fprintf(debug, "pmegrid %d %d %d max thread pmegrid %d %d %d\n",
nx, ny, nz,
nst[XX], nst[YY], nst[ZZ]);
}
snew(grids->grid_th, grids->nthread);
t = 0;
gridsize = nst[XX]*nst[YY]*nst[ZZ];
set_gridsize_alignment(&gridsize, pme_order);
snew_aligned(grids->grid_all,
grids->nthread*gridsize+(grids->nthread+1)*GMX_CACHE_SEP,
SIMD4_ALIGNMENT);
for (x = 0; x < grids->nc[XX]; x++)
{
for (y = 0; y < grids->nc[YY]; y++)
{
for (z = 0; z < grids->nc[ZZ]; z++)
{
pmegrid_init(&grids->grid_th[t],
x, y, z,
(n[XX]*(x ))/grids->nc[XX],
(n[YY]*(y ))/grids->nc[YY],
(n[ZZ]*(z ))/grids->nc[ZZ],
(n[XX]*(x+1))/grids->nc[XX],
(n[YY]*(y+1))/grids->nc[YY],
(n[ZZ]*(z+1))/grids->nc[ZZ],
TRUE,
pme_order,
grids->grid_all+GMX_CACHE_SEP+t*(gridsize+GMX_CACHE_SEP));
t++;
}
}
}
}
else
{
grids->grid_th = NULL;
}
snew(grids->g2t, DIM);
tfac = 1;
for (d = DIM-1; d >= 0; d--)
{
snew(grids->g2t[d], n[d]);
t = 0;
for (i = 0; i < n[d]; i++)
{
/* The second check should match the parameters
* of the pmegrid_init call above.
*/
while (t + 1 < grids->nc[d] && i >= (n[d]*(t+1))/grids->nc[d])
{
t++;
}
grids->g2t[d][i] = t*tfac;
}
tfac *= grids->nc[d];
switch (d)
{
case XX: max_comm_lines = overlap_x; break;
case YY: max_comm_lines = overlap_y; break;
case ZZ: max_comm_lines = pme_order - 1; break;
}
grids->nthread_comm[d] = 0;
while ((n[d]*grids->nthread_comm[d])/grids->nc[d] < max_comm_lines &&
grids->nthread_comm[d] < grids->nc[d])
{
grids->nthread_comm[d]++;
}
if (debug != NULL)
{
fprintf(debug, "pmegrid thread grid communication range in %c: %d\n",
'x'+d, grids->nthread_comm[d]);
}
/* It should be possible to make grids->nthread_comm[d]==grids->nc[d]
* work, but this is not a problematic restriction.
*/
if (grids->nc[d] > 1 && grids->nthread_comm[d] > grids->nc[d])
{
gmx_fatal(FARGS, "Too many threads for PME (%d) compared to the number of grid lines, reduce the number of threads doing PME", grids->nthread);
}
}
}
static int mk_grey(egCol egc[], t_graph *g, int *AtomI,
int npbcdim, matrix box, const rvec x[], int *nerror)
{
int m, j, ng, ai, aj, g0;
rvec dx, hbox;
gmx_bool bTriclinic;
ivec is_aj;
t_pbc pbc;
for (m = 0; (m < DIM); m++)
{
hbox[m] = box[m][m]*0.5;
}
bTriclinic = TRICLINIC(box);
g0 = g->at_start;
ng = 0;
ai = g0 + *AtomI;
/* Loop over all the bonds */
for (j = 0; (j < g->nedge[ai-g0]); j++)
{
aj = g->edge[ai-g0][j];
/* If there is a white one, make it grey and set pbc */
if (g->bScrewPBC)
{
mk_1shift_screw(box, hbox, x[ai], x[aj], g->ishift[ai], is_aj);
}
else if (bTriclinic)
{
mk_1shift_tric(npbcdim, box, hbox, x[ai], x[aj], g->ishift[ai], is_aj);
}
else
{
mk_1shift(npbcdim, hbox, x[ai], x[aj], g->ishift[ai], is_aj);
}
if (egc[aj-g0] == egcolWhite)
{
if (aj - g0 < *AtomI)
{
*AtomI = aj - g0;
}
egc[aj-g0] = egcolGrey;
copy_ivec(is_aj, g->ishift[aj]);
ng++;
}
else if ((is_aj[XX] != g->ishift[aj][XX]) ||
(is_aj[YY] != g->ishift[aj][YY]) ||
(is_aj[ZZ] != g->ishift[aj][ZZ]))
{
if (gmx_debug_at)
{
set_pbc(&pbc, -1, box);
pbc_dx(&pbc, x[ai], x[aj], dx);
fprintf(debug, "mk_grey: shifts for atom %d due to atom %d\n"
"are (%d,%d,%d), should be (%d,%d,%d)\n"
"dx = (%g,%g,%g)\n",
aj+1, ai+1, is_aj[XX], is_aj[YY], is_aj[ZZ],
g->ishift[aj][XX], g->ishift[aj][YY], g->ishift[aj][ZZ],
dx[XX], dx[YY], dx[ZZ]);
}
(*nerror)++;
}
}
return ng;
}
int gmx_pme_recv_coeffs_coords(struct gmx_pme_pp *pme_pp,
int *natoms,
real **chargeA,
real **chargeB,
real **sqrt_c6A,
real **sqrt_c6B,
real **sigmaA,
real **sigmaB,
matrix box,
rvec **x,
rvec **f,
int *maxshift_x,
int *maxshift_y,
gmx_bool *bFreeEnergy_q,
gmx_bool *bFreeEnergy_lj,
real *lambda_q,
real *lambda_lj,
gmx_bool *bEnerVir,
int *pme_flags,
gmx_int64_t *step,
ivec grid_size,
real *ewaldcoeff_q,
real *ewaldcoeff_lj)
{
int nat = 0, status;
*pme_flags = 0;
#ifdef GMX_MPI
gmx_pme_comm_n_box_t cnb;
int messages;
cnb.flags = 0;
messages = 0;
do
{
/* Receive the send count, box and time step from the peer PP node */
MPI_Recv(&cnb, sizeof(cnb), MPI_BYTE,
pme_pp->node_peer, eCommType_CNB,
pme_pp->mpi_comm_mysim, MPI_STATUS_IGNORE);
if (debug)
{
fprintf(debug, "PME only rank receiving:%s%s%s%s%s\n",
(cnb.flags & PP_PME_CHARGE) ? " charges" : "",
(cnb.flags & PP_PME_COORD ) ? " coordinates" : "",
(cnb.flags & PP_PME_FINISH) ? " finish" : "",
(cnb.flags & PP_PME_SWITCHGRID) ? " switch grid" : "",
(cnb.flags & PP_PME_RESETCOUNTERS) ? " reset counters" : "");
}
if (cnb.flags & PP_PME_SWITCHGRID)
{
/* Special case, receive the new parameters and return */
copy_ivec(cnb.grid_size, grid_size);
*ewaldcoeff_q = cnb.ewaldcoeff_q;
*ewaldcoeff_lj = cnb.ewaldcoeff_lj;
return pmerecvqxSWITCHGRID;
}
if (cnb.flags & PP_PME_RESETCOUNTERS)
{
/* Special case, receive the step and return */
*step = cnb.step;
return pmerecvqxRESETCOUNTERS;
}
if (cnb.flags & (PP_PME_CHARGE | PP_PME_SQRTC6 | PP_PME_SIGMA))
{
/* Receive the send counts from the other PP nodes */
for (int sender = 0; sender < pme_pp->nnode; sender++)
{
if (pme_pp->node[sender] == pme_pp->node_peer)
{
pme_pp->nat[sender] = cnb.natoms;
}
else
{
MPI_Irecv(&(pme_pp->nat[sender]), sizeof(pme_pp->nat[0]),
MPI_BYTE,
pme_pp->node[sender], eCommType_CNB,
pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
}
}
MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
messages = 0;
nat = 0;
for (int sender = 0; sender < pme_pp->nnode; sender++)
{
nat += pme_pp->nat[sender];
}
if (nat > pme_pp->nalloc)
{
pme_pp->nalloc = over_alloc_dd(nat);
if (cnb.flags & PP_PME_CHARGE)
{
srenew(pme_pp->chargeA, pme_pp->nalloc);
}
if (cnb.flags & PP_PME_CHARGEB)
{
srenew(pme_pp->chargeB, pme_pp->nalloc);
}
if (cnb.flags & PP_PME_SQRTC6)
{
srenew(pme_pp->sqrt_c6A, pme_pp->nalloc);
}
if (cnb.flags & PP_PME_SQRTC6B)
{
srenew(pme_pp->sqrt_c6B, pme_pp->nalloc);
}
if (cnb.flags & PP_PME_SIGMA)
{
srenew(pme_pp->sigmaA, pme_pp->nalloc);
}
if (cnb.flags & PP_PME_SIGMAB)
{
srenew(pme_pp->sigmaB, pme_pp->nalloc);
}
srenew(pme_pp->x, pme_pp->nalloc);
srenew(pme_pp->f, pme_pp->nalloc);
}
/* maxshift is sent when the charges are sent */
*maxshift_x = cnb.maxshift_x;
*maxshift_y = cnb.maxshift_y;
/* Receive the charges in place */
for (int q = 0; q < eCommType_NR; q++)
{
real *charge_pp;
if (!(cnb.flags & (PP_PME_CHARGE<<q)))
{
continue;
}
switch (q)
{
case eCommType_ChargeA: charge_pp = pme_pp->chargeA; break;
case eCommType_ChargeB: charge_pp = pme_pp->chargeB; break;
case eCommType_SQRTC6A: charge_pp = pme_pp->sqrt_c6A; break;
case eCommType_SQRTC6B: charge_pp = pme_pp->sqrt_c6B; break;
case eCommType_SigmaA: charge_pp = pme_pp->sigmaA; break;
case eCommType_SigmaB: charge_pp = pme_pp->sigmaB; break;
default: gmx_incons("Wrong eCommType");
}
nat = 0;
for (int sender = 0; sender < pme_pp->nnode; sender++)
{
if (pme_pp->nat[sender] > 0)
{
MPI_Irecv(charge_pp+nat,
pme_pp->nat[sender]*sizeof(real),
MPI_BYTE,
pme_pp->node[sender], q,
pme_pp->mpi_comm_mysim,
&pme_pp->req[messages++]);
nat += pme_pp->nat[sender];
if (debug)
{
fprintf(debug, "Received from PP rank %d: %d %s\n",
pme_pp->node[sender], pme_pp->nat[sender],
(q == eCommType_ChargeA ||
q == eCommType_ChargeB) ? "charges" : "params");
}
}
}
}
pme_pp->flags_charge = cnb.flags;
}
if (cnb.flags & PP_PME_COORD)
{
if (!(pme_pp->flags_charge & (PP_PME_CHARGE | PP_PME_SQRTC6)))
{
gmx_incons("PME-only rank received coordinates before charges and/or C6-values"
);
}
/* The box, FE flag and lambda are sent along with the coordinates
* */
copy_mat(cnb.box, box);
*bFreeEnergy_q = ((cnb.flags & GMX_PME_DO_COULOMB) &&
(cnb.flags & PP_PME_FEP_Q));
*bFreeEnergy_lj = ((cnb.flags & GMX_PME_DO_LJ) &&
(cnb.flags & PP_PME_FEP_LJ));
*lambda_q = cnb.lambda_q;
*lambda_lj = cnb.lambda_lj;
*bEnerVir = (cnb.flags & PP_PME_ENER_VIR);
*pme_flags = cnb.flags;
if (*bFreeEnergy_q && !(pme_pp->flags_charge & PP_PME_CHARGEB))
{
gmx_incons("PME-only rank received free energy request, but "
"did not receive B-state charges");
}
if (*bFreeEnergy_lj && !(pme_pp->flags_charge & PP_PME_SQRTC6B))
{
gmx_incons("PME-only rank received free energy request, but "
"did not receive B-state C6-values");
}
/* Receive the coordinates in place */
nat = 0;
for (int sender = 0; sender < pme_pp->nnode; sender++)
{
if (pme_pp->nat[sender] > 0)
{
MPI_Irecv(pme_pp->x[nat], pme_pp->nat[sender]*sizeof(rvec),
MPI_BYTE,
pme_pp->node[sender], eCommType_COORD,
pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
nat += pme_pp->nat[sender];
if (debug)
{
fprintf(debug, "Received from PP rank %d: %d "
"coordinates\n",
pme_pp->node[sender], pme_pp->nat[sender]);
}
}
}
}
/* Wait for the coordinates and/or charges to arrive */
MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
messages = 0;
}
while (!(cnb.flags & (PP_PME_COORD | PP_PME_FINISH)));
status = ((cnb.flags & PP_PME_FINISH) ? pmerecvqxFINISH : pmerecvqxX);
*step = cnb.step;
#else
GMX_UNUSED_VALUE(box);
GMX_UNUSED_VALUE(maxshift_x);
GMX_UNUSED_VALUE(maxshift_y);
GMX_UNUSED_VALUE(bFreeEnergy_q);
GMX_UNUSED_VALUE(bFreeEnergy_lj);
GMX_UNUSED_VALUE(lambda_q);
GMX_UNUSED_VALUE(lambda_lj);
GMX_UNUSED_VALUE(bEnerVir);
GMX_UNUSED_VALUE(step);
GMX_UNUSED_VALUE(grid_size);
GMX_UNUSED_VALUE(ewaldcoeff_q);
GMX_UNUSED_VALUE(ewaldcoeff_lj);
status = pmerecvqxX;
#endif
*natoms = nat;
*chargeA = pme_pp->chargeA;
*chargeB = pme_pp->chargeB;
*sqrt_c6A = pme_pp->sqrt_c6A;
*sqrt_c6B = pme_pp->sqrt_c6B;
*sigmaA = pme_pp->sigmaA;
*sigmaB = pme_pp->sigmaB;
*x = pme_pp->x;
*f = pme_pp->f;
return status;
}
/*! \brief Called by PME-only ranks to receive coefficients and coordinates
*
* \param[in,out] pme_pp PME-PP communication structure.
* \param[out] natoms Number of received atoms.
* \param[out] box System box, if received.
* \param[out] maxshift_x Maximum shift in X direction, if received.
* \param[out] maxshift_y Maximum shift in Y direction, if received.
* \param[out] lambda_q Free-energy lambda for electrostatics, if received.
* \param[out] lambda_lj Free-energy lambda for Lennard-Jones, if received.
* \param[out] bEnerVir Set to true if this is an energy/virial calculation step, otherwise set to false.
* \param[out] step MD integration step number.
* \param[out] grid_size PME grid size, if received.
* \param[out] ewaldcoeff_q Ewald cut-off parameter for electrostatics, if received.
* \param[out] ewaldcoeff_lj Ewald cut-off parameter for Lennard-Jones, if received.
* \param[out] atomSetChanged Set to true only if the local domain atom data (charges/coefficients)
* has been received (after DD) and should be reinitialized. Otherwise not changed.
*
* \retval pmerecvqxX All parameters were set, chargeA and chargeB can be NULL.
* \retval pmerecvqxFINISH No parameters were set.
* \retval pmerecvqxSWITCHGRID Only grid_size and *ewaldcoeff were set.
* \retval pmerecvqxRESETCOUNTERS *step was set.
*/
static int gmx_pme_recv_coeffs_coords(gmx_pme_pp *pme_pp,
int *natoms,
matrix box,
int *maxshift_x,
int *maxshift_y,
real *lambda_q,
real *lambda_lj,
gmx_bool *bEnerVir,
int64_t *step,
ivec *grid_size,
real *ewaldcoeff_q,
real *ewaldcoeff_lj,
bool *atomSetChanged)
{
int status = -1;
int nat = 0;
#if GMX_MPI
unsigned int flags = 0;
int messages = 0;
do
{
gmx_pme_comm_n_box_t cnb;
cnb.flags = 0;
/* Receive the send count, box and time step from the peer PP node */
MPI_Recv(&cnb, sizeof(cnb), MPI_BYTE,
pme_pp->peerRankId, eCommType_CNB,
pme_pp->mpi_comm_mysim, MPI_STATUS_IGNORE);
/* We accumulate all received flags */
flags |= cnb.flags;
*step = cnb.step;
if (debug)
{
fprintf(debug, "PME only rank receiving:%s%s%s%s%s\n",
(cnb.flags & PP_PME_CHARGE) ? " charges" : "",
(cnb.flags & PP_PME_COORD ) ? " coordinates" : "",
(cnb.flags & PP_PME_FINISH) ? " finish" : "",
(cnb.flags & PP_PME_SWITCHGRID) ? " switch grid" : "",
(cnb.flags & PP_PME_RESETCOUNTERS) ? " reset counters" : "");
}
if (cnb.flags & PP_PME_FINISH)
{
status = pmerecvqxFINISH;
}
if (cnb.flags & PP_PME_SWITCHGRID)
{
/* Special case, receive the new parameters and return */
copy_ivec(cnb.grid_size, *grid_size);
*ewaldcoeff_q = cnb.ewaldcoeff_q;
*ewaldcoeff_lj = cnb.ewaldcoeff_lj;
status = pmerecvqxSWITCHGRID;
}
if (cnb.flags & PP_PME_RESETCOUNTERS)
{
/* Special case, receive the step (set above) and return */
status = pmerecvqxRESETCOUNTERS;
}
if (cnb.flags & (PP_PME_CHARGE | PP_PME_SQRTC6 | PP_PME_SIGMA))
{
*atomSetChanged = true;
/* Receive the send counts from the other PP nodes */
for (auto &sender : pme_pp->ppRanks)
{
if (sender.rankId == pme_pp->peerRankId)
{
sender.numAtoms = cnb.natoms;
}
else
{
MPI_Irecv(&sender.numAtoms, sizeof(sender.numAtoms),
MPI_BYTE,
sender.rankId, eCommType_CNB,
pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
}
}
MPI_Waitall(messages, pme_pp->req.data(), pme_pp->stat.data());
messages = 0;
nat = 0;
for (const auto &sender : pme_pp->ppRanks)
{
nat += sender.numAtoms;
}
if (cnb.flags & PP_PME_CHARGE)
{
pme_pp->chargeA.resizeWithPadding(nat);
}
if (cnb.flags & PP_PME_CHARGEB)
{
pme_pp->chargeB.resize(nat);
}
if (cnb.flags & PP_PME_SQRTC6)
{
pme_pp->sqrt_c6A.resize(nat);
}
if (cnb.flags & PP_PME_SQRTC6B)
{
pme_pp->sqrt_c6B.resize(nat);
}
if (cnb.flags & PP_PME_SIGMA)
{
pme_pp->sigmaA.resize(nat);
}
if (cnb.flags & PP_PME_SIGMAB)
{
pme_pp->sigmaB.resize(nat);
}
pme_pp->x.resizeWithPadding(nat);
pme_pp->f.resize(nat);
/* maxshift is sent when the charges are sent */
*maxshift_x = cnb.maxshift_x;
*maxshift_y = cnb.maxshift_y;
/* Receive the charges in place */
for (int q = 0; q < eCommType_NR; q++)
{
real *bufferPtr;
if (!(cnb.flags & (PP_PME_CHARGE<<q)))
{
continue;
}
switch (q)
{
case eCommType_ChargeA: bufferPtr = pme_pp->chargeA.data(); break;
case eCommType_ChargeB: bufferPtr = pme_pp->chargeB.data(); break;
case eCommType_SQRTC6A: bufferPtr = pme_pp->sqrt_c6A.data(); break;
case eCommType_SQRTC6B: bufferPtr = pme_pp->sqrt_c6B.data(); break;
case eCommType_SigmaA: bufferPtr = pme_pp->sigmaA.data(); break;
case eCommType_SigmaB: bufferPtr = pme_pp->sigmaB.data(); break;
default: gmx_incons("Wrong eCommType");
}
nat = 0;
for (const auto &sender : pme_pp->ppRanks)
{
if (sender.numAtoms > 0)
{
MPI_Irecv(bufferPtr+nat,
sender.numAtoms*sizeof(real),
MPI_BYTE,
sender.rankId, q,
pme_pp->mpi_comm_mysim,
&pme_pp->req[messages++]);
nat += sender.numAtoms;
if (debug)
{
fprintf(debug, "Received from PP rank %d: %d %s\n",
sender.rankId, sender.numAtoms,
(q == eCommType_ChargeA ||
q == eCommType_ChargeB) ? "charges" : "params");
}
}
}
}
}
if (cnb.flags & PP_PME_COORD)
{
/* The box, FE flag and lambda are sent along with the coordinates
* */
copy_mat(cnb.box, box);
*lambda_q = cnb.lambda_q;
*lambda_lj = cnb.lambda_lj;
*bEnerVir = ((cnb.flags & PP_PME_ENER_VIR) != 0u);
*step = cnb.step;
/* Receive the coordinates in place */
nat = 0;
for (const auto &sender : pme_pp->ppRanks)
{
if (sender.numAtoms > 0)
{
MPI_Irecv(pme_pp->x[nat],
sender.numAtoms*sizeof(rvec),
MPI_BYTE,
sender.rankId, eCommType_COORD,
pme_pp->mpi_comm_mysim, &pme_pp->req[messages++]);
nat += sender.numAtoms;
if (debug)
{
fprintf(debug, "Received from PP rank %d: %d "
"coordinates\n",
sender.rankId, sender.numAtoms);
}
}
}
status = pmerecvqxX;
}
/* Wait for the coordinates and/or charges to arrive */
MPI_Waitall(messages, pme_pp->req.data(), pme_pp->stat.data());
messages = 0;
}
while (status == -1);
#else
GMX_UNUSED_VALUE(pme_pp);
GMX_UNUSED_VALUE(box);
GMX_UNUSED_VALUE(maxshift_x);
GMX_UNUSED_VALUE(maxshift_y);
GMX_UNUSED_VALUE(lambda_q);
GMX_UNUSED_VALUE(lambda_lj);
GMX_UNUSED_VALUE(bEnerVir);
GMX_UNUSED_VALUE(step);
GMX_UNUSED_VALUE(grid_size);
GMX_UNUSED_VALUE(ewaldcoeff_q);
GMX_UNUSED_VALUE(ewaldcoeff_lj);
GMX_UNUSED_VALUE(atomSetChanged);
status = pmerecvqxX;
#endif
if (status == pmerecvqxX)
{
*natoms = nat;
}
return status;
}
int gmx_pme_recv_q_x(struct gmx_pme_pp *pme_pp,
real **chargeA, real **chargeB,
matrix box, rvec **x,rvec **f,
int *maxshift_x, int *maxshift_y,
gmx_bool *bFreeEnergy,real *lambda,
gmx_bool *bEnerVir,
gmx_large_int_t *step,
ivec grid_size, real *ewaldcoeff)
{
gmx_pme_comm_n_box_t cnb;
int nat=0,q,messages,sender;
real *charge_pp;
messages = 0;
/* avoid compiler warning about unused variable without MPI support */
cnb.flags = 0;
#ifdef GMX_MPI
do {
/* Receive the send count, box and time step from the peer PP node */
MPI_Recv(&cnb,sizeof(cnb),MPI_BYTE,
pme_pp->node_peer,0,
pme_pp->mpi_comm_mysim,MPI_STATUS_IGNORE);
if (debug)
{
fprintf(debug,"PME only node receiving:%s%s%s%s\n",
(cnb.flags & PP_PME_CHARGE) ? " charges" : "",
(cnb.flags & PP_PME_COORD ) ? " coordinates" : "",
(cnb.flags & PP_PME_FINISH) ? " finish" : "",
(cnb.flags & PP_PME_SWITCH) ? " switch" : "");
}
if (cnb.flags & PP_PME_SWITCH)
{
/* Special case, receive the new parameters and return */
copy_ivec(cnb.grid_size,grid_size);
*ewaldcoeff = cnb.ewaldcoeff;
return -2;
}
if (cnb.flags & PP_PME_CHARGE) {
/* Receive the send counts from the other PP nodes */
for(sender=0; sender<pme_pp->nnode; sender++) {
if (pme_pp->node[sender] == pme_pp->node_peer) {
pme_pp->nat[sender] = cnb.natoms;
} else {
MPI_Irecv(&(pme_pp->nat[sender]),sizeof(pme_pp->nat[0]),
MPI_BYTE,
pme_pp->node[sender],0,
pme_pp->mpi_comm_mysim,&pme_pp->req[messages++]);
}
}
MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
messages = 0;
nat = 0;
for(sender=0; sender<pme_pp->nnode; sender++)
nat += pme_pp->nat[sender];
if (nat > pme_pp->nalloc) {
pme_pp->nalloc = over_alloc_dd(nat);
srenew(pme_pp->chargeA,pme_pp->nalloc);
if (cnb.flags & PP_PME_CHARGEB)
srenew(pme_pp->chargeB,pme_pp->nalloc);
srenew(pme_pp->x,pme_pp->nalloc);
srenew(pme_pp->f,pme_pp->nalloc);
}
/* maxshift is sent when the charges are sent */
*maxshift_x = cnb.maxshift_x;
*maxshift_y = cnb.maxshift_y;
/* Receive the charges in place */
for(q=0; q<((cnb.flags & PP_PME_CHARGEB) ? 2 : 1); q++) {
if (q == 0)
charge_pp = pme_pp->chargeA;
else
charge_pp = pme_pp->chargeB;
nat = 0;
for(sender=0; sender<pme_pp->nnode; sender++) {
if (pme_pp->nat[sender] > 0) {
MPI_Irecv(charge_pp+nat,
pme_pp->nat[sender]*sizeof(real),
MPI_BYTE,
pme_pp->node[sender],1+q,
pme_pp->mpi_comm_mysim,
&pme_pp->req[messages++]);
nat += pme_pp->nat[sender];
if (debug)
fprintf(debug,"Received from PP node %d: %d "
"charges\n",
pme_pp->node[sender],pme_pp->nat[sender]);
}
}
}
pme_pp->flags_charge = cnb.flags;
}
if (cnb.flags & PP_PME_COORD) {
if (!(pme_pp->flags_charge & PP_PME_CHARGE))
gmx_incons("PME-only node received coordinates before charges"
);
/* The box, FE flag and lambda are sent along with the coordinates
* */
copy_mat(cnb.box,box);
*bFreeEnergy = (cnb.flags & PP_PME_FEP);
*lambda = cnb.lambda;
*bEnerVir = (cnb.flags & PP_PME_ENER_VIR);
if (*bFreeEnergy && !(pme_pp->flags_charge & PP_PME_CHARGEB))
gmx_incons("PME-only node received free energy request, but "
"did not receive B-state charges");
/* Receive the coordinates in place */
nat = 0;
for(sender=0; sender<pme_pp->nnode; sender++) {
if (pme_pp->nat[sender] > 0) {
MPI_Irecv(pme_pp->x[nat],pme_pp->nat[sender]*sizeof(rvec),
MPI_BYTE,
pme_pp->node[sender],3,
pme_pp->mpi_comm_mysim,&pme_pp->req[messages++]);
nat += pme_pp->nat[sender];
if (debug)
fprintf(debug,"Received from PP node %d: %d "
"coordinates\n",
pme_pp->node[sender],pme_pp->nat[sender]);
}
}
}
/* Wait for the coordinates and/or charges to arrive */
MPI_Waitall(messages, pme_pp->req, pme_pp->stat);
messages = 0;
} while (!(cnb.flags & (PP_PME_COORD | PP_PME_FINISH)));
*step = cnb.step;
#endif
*chargeA = pme_pp->chargeA;
*chargeB = pme_pp->chargeB;
*x = pme_pp->x;
*f = pme_pp->f;
return ((cnb.flags & PP_PME_FINISH) ? -1 : nat);
}
int psize (t_topology *top, atom_id *index, int isize, rvec *x, t_PolKey *param, gmx_bool bCG, gmx_bool bFocus) {
int i,j;
rvec minlen, maxlen;
rvec olen, clen, flen, cen;
ivec n, np, tn, nsmall;
real nsmem, gmem, zofac;
real r, np_float;
minlen[XX] = 9999; minlen[YY] = 9999; minlen[ZZ] = 9999;
maxlen[XX] = -9999; maxlen[YY] = -9999; maxlen[ZZ] = -9999;
for (i=0;i<isize;i++) {
r = top->atoms.pdbinfo[index[i]].bfac;
for(j=0;j<DIM;j++) {
if((x[index[i]][j]*10)-r < minlen[j])
minlen[j] = (x[index[i]][j]*10) - r;
if((x[index[i]][j]*10)+r > maxlen[j])
maxlen[j] = (x[index[i]][j]*10) + r;
}
}
for (i=0;i<DIM;i++) {
olen[i] = maxlen[i] - minlen[i];
clen[i] = param->cfac * olen[i];
flen[i] = param->fadd + olen[i];
if(flen[i]>clen[i])
flen[i] = clen[i];
cen[i] = (maxlen[i] + minlen[i])/2;
tn[i] = (int) flen[i]/param->gridspace + 0.5;
n[i] = 32*((int)((tn[i] - 1) / 32.0 + 0.5)) + 1;
nsmall[i] = 32*((int)((tn[i] - 1) / 32.0 + 0.5)) + 1;
if (nsmall[i] < 33)
nsmall[i] = 33;
}
//To Check the available memory
gmem = 200.0 * n[XX] * n[YY] * n[ZZ] / 1024 / 1024;
while(1) {
nsmem = 200.0 * nsmall[XX] * nsmall[YY] * nsmall[ZZ] / 1024 / 1024;
if(nsmem<param->gmemceil)
break;
else {
i = maxIndex(nsmall);
nsmall[i] = 32 * ((nsmall[i] - 1)/32 - 1) + 1;
if (nsmall <= 0) {
gmx_fatal(FARGS, "You picked a memory ceiling that is too small\n");
}
}
}
// Calculating pdime => np
if (gmem >= param->gmemceil) {
zofac = 1 + 2 * param->ofrac;
for (i=0;i<DIM;i++) {
np_float = n[i]/(float)nsmall[i];
if (np_float > 1)
np[i] = (int)(zofac*n[1]/nsmall[i] + 1.0);
}
}
if(gmem >= param->gmemceil)
param->mg_type = mg_para;
else
param->mg_type = mg_auto;
if (bCG) {
copy_rvec(clen,param->cglen);
copy_rvec(cen,param->cgcent);
}
if(!bFocus) {
copy_rvec(flen,param->fglen);
copy_rvec(cen,param->fgcent);
}
copy_ivec(nsmall,param->dime);
if(param->mg_type == mg_para)
copy_ivec(np, param->pdime);
return 0;
}
int pbc_dx_aiuc(const t_pbc *pbc, const rvec x1, const rvec x2, rvec dx)
{
int i, j, is;
rvec dx_start, trial;
real d2min, d2trial;
ivec ishift, ishift_start;
rvec_sub(x1, x2, dx);
clear_ivec(ishift);
switch (pbc->ePBCDX)
{
case epbcdxRECTANGULAR:
for (i = 0; i < DIM; i++)
{
if (dx[i] > pbc->hbox_diag[i])
{
dx[i] -= pbc->fbox_diag[i];
ishift[i]--;
}
else if (dx[i] <= pbc->mhbox_diag[i])
{
dx[i] += pbc->fbox_diag[i];
ishift[i]++;
}
}
break;
case epbcdxTRICLINIC:
/* For triclinic boxes the performance difference between
* if/else and two while loops is negligible.
* However, the while version can cause extreme delays
* before a simulation crashes due to large forces which
* can cause unlimited displacements.
* Also allowing multiple shifts would index fshift beyond bounds.
*/
for (i = DIM-1; i >= 1; i--)
{
if (dx[i] > pbc->hbox_diag[i])
{
for (j = i; j >= 0; j--)
{
dx[j] -= pbc->box[i][j];
}
ishift[i]--;
}
else if (dx[i] <= pbc->mhbox_diag[i])
{
for (j = i; j >= 0; j--)
{
dx[j] += pbc->box[i][j];
}
ishift[i]++;
}
}
/* Allow 2 shifts in x */
if (dx[XX] > pbc->hbox_diag[XX])
{
dx[XX] -= pbc->fbox_diag[XX];
ishift[XX]--;
if (dx[XX] > pbc->hbox_diag[XX])
{
dx[XX] -= pbc->fbox_diag[XX];
ishift[XX]--;
}
}
else if (dx[XX] <= pbc->mhbox_diag[XX])
{
dx[XX] += pbc->fbox_diag[XX];
ishift[XX]++;
if (dx[XX] <= pbc->mhbox_diag[XX])
{
dx[XX] += pbc->fbox_diag[XX];
ishift[XX]++;
}
}
/* dx is the distance in a rectangular box */
d2min = norm2(dx);
if (d2min > pbc->max_cutoff2)
{
copy_rvec(dx, dx_start);
copy_ivec(ishift, ishift_start);
d2min = norm2(dx);
/* Now try all possible shifts, when the distance is within max_cutoff
* it must be the shortest possible distance.
*/
i = 0;
while ((d2min > pbc->max_cutoff2) && (i < pbc->ntric_vec))
{
rvec_add(dx_start, pbc->tric_vec[i], trial);
d2trial = norm2(trial);
if (d2trial < d2min)
{
copy_rvec(trial, dx);
ivec_add(ishift_start, pbc->tric_shift[i], ishift);
d2min = d2trial;
}
i++;
}
}
break;
case epbcdx2D_RECT:
for (i = 0; i < DIM; i++)
{
if (i != pbc->dim)
{
if (dx[i] > pbc->hbox_diag[i])
{
dx[i] -= pbc->fbox_diag[i];
ishift[i]--;
}
else if (dx[i] <= pbc->mhbox_diag[i])
{
dx[i] += pbc->fbox_diag[i];
ishift[i]++;
}
}
}
break;
case epbcdx2D_TRIC:
d2min = 0;
for (i = DIM-1; i >= 1; i--)
{
if (i != pbc->dim)
{
if (dx[i] > pbc->hbox_diag[i])
{
for (j = i; j >= 0; j--)
{
dx[j] -= pbc->box[i][j];
}
ishift[i]--;
}
else if (dx[i] <= pbc->mhbox_diag[i])
{
for (j = i; j >= 0; j--)
{
dx[j] += pbc->box[i][j];
}
ishift[i]++;
}
d2min += dx[i]*dx[i];
}
}
if (pbc->dim != XX)
{
/* Allow 2 shifts in x */
if (dx[XX] > pbc->hbox_diag[XX])
{
dx[XX] -= pbc->fbox_diag[XX];
ishift[XX]--;
if (dx[XX] > pbc->hbox_diag[XX])
{
dx[XX] -= pbc->fbox_diag[XX];
ishift[XX]--;
}
}
else if (dx[XX] <= pbc->mhbox_diag[XX])
{
dx[XX] += pbc->fbox_diag[XX];
ishift[XX]++;
if (dx[XX] <= pbc->mhbox_diag[XX])
{
dx[XX] += pbc->fbox_diag[XX];
ishift[XX]++;
}
}
d2min += dx[XX]*dx[XX];
}
if (d2min > pbc->max_cutoff2)
{
copy_rvec(dx, dx_start);
copy_ivec(ishift, ishift_start);
/* Now try all possible shifts, when the distance is within max_cutoff
* it must be the shortest possible distance.
*/
i = 0;
while ((d2min > pbc->max_cutoff2) && (i < pbc->ntric_vec))
{
rvec_add(dx_start, pbc->tric_vec[i], trial);
d2trial = 0;
for (j = 0; j < DIM; j++)
{
if (j != pbc->dim)
{
d2trial += trial[j]*trial[j];
}
}
if (d2trial < d2min)
{
copy_rvec(trial, dx);
ivec_add(ishift_start, pbc->tric_shift[i], ishift);
d2min = d2trial;
}
i++;
}
}
break;
case epbcdx1D_RECT:
i = pbc->dim;
if (dx[i] > pbc->hbox_diag[i])
{
dx[i] -= pbc->fbox_diag[i];
ishift[i]--;
}
else if (dx[i] <= pbc->mhbox_diag[i])
{
dx[i] += pbc->fbox_diag[i];
ishift[i]++;
}
break;
case epbcdx1D_TRIC:
i = pbc->dim;
if (dx[i] > pbc->hbox_diag[i])
{
rvec_dec(dx, pbc->box[i]);
ishift[i]--;
}
else if (dx[i] <= pbc->mhbox_diag[i])
{
rvec_inc(dx, pbc->box[i]);
ishift[i]++;
}
break;
case epbcdxSCREW_RECT:
/* The shift definition requires x first */
if (dx[XX] > pbc->hbox_diag[XX])
{
dx[XX] -= pbc->fbox_diag[XX];
ishift[XX]--;
}
else if (dx[XX] <= pbc->mhbox_diag[XX])
{
dx[XX] += pbc->fbox_diag[XX];
ishift[XX]++;
}
if (ishift[XX] == 1 || ishift[XX] == -1)
{
/* Rotate around the x-axis in the middle of the box */
dx[YY] = pbc->box[YY][YY] - x1[YY] - x2[YY];
dx[ZZ] = pbc->box[ZZ][ZZ] - x1[ZZ] - x2[ZZ];
}
/* Normal pbc for y and z */
for (i = YY; i <= ZZ; i++)
{
if (dx[i] > pbc->hbox_diag[i])
{
dx[i] -= pbc->fbox_diag[i];
ishift[i]--;
}
else if (dx[i] <= pbc->mhbox_diag[i])
{
dx[i] += pbc->fbox_diag[i];
ishift[i]++;
}
}
break;
case epbcdxNOPBC:
case epbcdxUNSUPPORTED:
break;
default:
gmx_fatal(FARGS, "Internal error in pbc_dx_aiuc, set_pbc_dd or set_pbc has not been called");
break;
}
is = IVEC2IS(ishift);
if (debug)
{
range_check_mesg(is, 0, SHIFTS, "PBC shift vector index range check.");
}
return is;
}
