Пример #1
0
int relax_shell_flexcon(FILE *fplog,t_commrec *cr,gmx_bool bVerbose,
			gmx_large_int_t mdstep,t_inputrec *inputrec,
			gmx_bool bDoNS,int force_flags,
			gmx_bool bStopCM,
			gmx_localtop_t *top,
			gmx_mtop_t* mtop,
			gmx_constr_t constr,
			gmx_enerdata_t *enerd,t_fcdata *fcd,
			t_state *state,rvec f[],
			tensor force_vir,
			t_mdatoms *md,
			t_nrnb *nrnb,gmx_wallcycle_t wcycle,
			t_graph *graph,
			gmx_groups_t *groups,
			struct gmx_shellfc *shfc,
			t_forcerec *fr,
			gmx_bool bBornRadii,
			double t,rvec mu_tot,
			int natoms,gmx_bool *bConverged,
			gmx_vsite_t *vsite,
			FILE *fp_field)
{
  int    nshell;
  t_shell *shell;
  t_idef *idef;
  rvec   *pos[2],*force[2],*acc_dir=NULL,*x_old=NULL;
  real   Epot[2],df[2];
  rvec   dx;
  real   sf_dir,invdt;
  real   ftol,xiH,xiS,dum=0;
  char   sbuf[22];
  gmx_bool   bCont,bInit;
  int    nat,dd_ac0,dd_ac1=0,i;
  int    start=md->start,homenr=md->homenr,end=start+homenr,cg0,cg1;
  int    nflexcon,g,number_steps,d,Min=0,count=0;
#define  Try (1-Min)             /* At start Try = 1 */

  bCont        = (mdstep == inputrec->init_step) && inputrec->bContinuation;
  bInit        = (mdstep == inputrec->init_step) || shfc->bRequireInit;
  ftol         = inputrec->em_tol;
  number_steps = inputrec->niter;
  nshell       = shfc->nshell;
  shell        = shfc->shell;
  nflexcon     = shfc->nflexcon;

  idef = &top->idef;

  if (DOMAINDECOMP(cr)) {
    nat = dd_natoms_vsite(cr->dd);
    if (nflexcon > 0) {
      dd_get_constraint_range(cr->dd,&dd_ac0,&dd_ac1);
      nat = max(nat,dd_ac1);
    }
  } else {
    nat = state->natoms;
  }

  if (nat > shfc->x_nalloc) {
    /* Allocate local arrays */
    shfc->x_nalloc = over_alloc_dd(nat);
    for(i=0; (i<2); i++) {
      srenew(shfc->x[i],shfc->x_nalloc);
      srenew(shfc->f[i],shfc->x_nalloc);
    }
  }
  for(i=0; (i<2); i++) {
    pos[i]   = shfc->x[i];
    force[i] = shfc->f[i];
  }
     
  /* With particle decomposition this code only works
   * when all particles involved with each shell are in the same cg.
   */

  if (bDoNS && inputrec->ePBC != epbcNONE && !DOMAINDECOMP(cr)) {
    /* This is the only time where the coordinates are used
     * before do_force is called, which normally puts all
     * charge groups in the box.
     */
    if (PARTDECOMP(cr)) {
      pd_cg_range(cr,&cg0,&cg1);
    } else {
      cg0 = 0;
      cg1 = top->cgs.nr;
    }
    put_charge_groups_in_box(fplog,cg0,cg1,fr->ePBC,state->box,
			     &(top->cgs),state->x,fr->cg_cm);
    if (graph)
      mk_mshift(fplog,graph,fr->ePBC,state->box,state->x);
  }

  /* After this all coordinate arrays will contain whole molecules */
  if (graph)
    shift_self(graph,state->box,state->x);

  if (nflexcon) {
    if (nat > shfc->flex_nalloc) {
      shfc->flex_nalloc = over_alloc_dd(nat);
      srenew(shfc->acc_dir,shfc->flex_nalloc);
      srenew(shfc->x_old,shfc->flex_nalloc);
    }
    acc_dir = shfc->acc_dir;
    x_old   = shfc->x_old;
    for(i=0; i<homenr; i++) {
      for(d=0; d<DIM; d++)
        shfc->x_old[i][d] =
	  state->x[start+i][d] - state->v[start+i][d]*inputrec->delta_t;
    }
  }

  /* Do a prediction of the shell positions */
  if (shfc->bPredict && !bCont) {
    predict_shells(fplog,state->x,state->v,inputrec->delta_t,nshell,shell,
		   md->massT,NULL,bInit);
  }

  /* do_force expected the charge groups to be in the box */
  if (graph)
    unshift_self(graph,state->box,state->x);

  /* Calculate the forces first time around */
  if (gmx_debug_at) {
    pr_rvecs(debug,0,"x b4 do_force",state->x + start,homenr);
  }
  do_force(fplog,cr,inputrec,mdstep,nrnb,wcycle,top,mtop,groups,
	   state->box,state->x,&state->hist,
	   force[Min],force_vir,md,enerd,fcd,
	   state->lambda,graph,
	   fr,vsite,mu_tot,t,fp_field,NULL,bBornRadii,
	   (bDoNS ? GMX_FORCE_NS : 0) | force_flags);

  sf_dir = 0;
  if (nflexcon) {
    init_adir(fplog,shfc,
	      constr,idef,inputrec,cr,dd_ac1,mdstep,md,start,end,
	      shfc->x_old-start,state->x,state->x,force[Min],
	      shfc->acc_dir-start,
	      fr->bMolPBC,state->box,state->lambda,&dum,nrnb);

    for(i=start; i<end; i++)
      sf_dir += md->massT[i]*norm2(shfc->acc_dir[i-start]);
  }

  Epot[Min] = enerd->term[F_EPOT];

  df[Min]=rms_force(cr,shfc->f[Min],nshell,shell,nflexcon,&sf_dir,&Epot[Min]);
  df[Try]=0;
  if (debug) {
    fprintf(debug,"df = %g  %g\n",df[Min],df[Try]);
  }

  if (gmx_debug_at) {
    pr_rvecs(debug,0,"force0",force[Min],md->nr);
  }

  if (nshell+nflexcon > 0) {
    /* Copy x to pos[Min] & pos[Try]: during minimization only the
     * shell positions are updated, therefore the other particles must
     * be set here.
     */
    memcpy(pos[Min],state->x,nat*sizeof(state->x[0]));
    memcpy(pos[Try],state->x,nat*sizeof(state->x[0]));
  }
  
  if (bVerbose && MASTER(cr))
    print_epot(stdout,mdstep,0,Epot[Min],df[Min],nflexcon,sf_dir);

  if (debug) {
    fprintf(debug,"%17s: %14.10e\n",
	    interaction_function[F_EKIN].longname,enerd->term[F_EKIN]);
    fprintf(debug,"%17s: %14.10e\n",
	    interaction_function[F_EPOT].longname,enerd->term[F_EPOT]);
    fprintf(debug,"%17s: %14.10e\n",
	    interaction_function[F_ETOT].longname,enerd->term[F_ETOT]);
    fprintf(debug,"SHELLSTEP %s\n",gmx_step_str(mdstep,sbuf));
  }
  
  /* First check whether we should do shells, or whether the force is 
   * low enough even without minimization.
   */
  *bConverged = (df[Min] < ftol);
  
  for(count=1; (!(*bConverged) && (count < number_steps)); count++) {
    if (vsite)
      construct_vsites(fplog,vsite,pos[Min],nrnb,inputrec->delta_t,state->v,
		       idef->iparams,idef->il,
		       fr->ePBC,fr->bMolPBC,graph,cr,state->box);
     
    if (nflexcon) {
      init_adir(fplog,shfc,
		constr,idef,inputrec,cr,dd_ac1,mdstep,md,start,end,
		x_old-start,state->x,pos[Min],force[Min],acc_dir-start,
		fr->bMolPBC,state->box,state->lambda,&dum,nrnb);
      
      directional_sd(fplog,pos[Min],pos[Try],acc_dir-start,start,end,
		     fr->fc_stepsize);
    }
    
    /* New positions, Steepest descent */
    shell_pos_sd(fplog,pos[Min],pos[Try],force[Min],nshell,shell,count); 

    /* do_force expected the charge groups to be in the box */
    if (graph)
      unshift_self(graph,state->box,pos[Try]);

    if (gmx_debug_at) {
      pr_rvecs(debug,0,"RELAX: pos[Min]  ",pos[Min] + start,homenr);
      pr_rvecs(debug,0,"RELAX: pos[Try]  ",pos[Try] + start,homenr);
    }
    /* Try the new positions */
    do_force(fplog,cr,inputrec,1,nrnb,wcycle,
	     top,mtop,groups,state->box,pos[Try],&state->hist,
	     force[Try],force_vir,
	     md,enerd,fcd,state->lambda,graph,
	     fr,vsite,mu_tot,t,fp_field,NULL,bBornRadii,
	     force_flags);
    
    if (gmx_debug_at) {
      pr_rvecs(debug,0,"RELAX: force[Min]",force[Min] + start,homenr);
      pr_rvecs(debug,0,"RELAX: force[Try]",force[Try] + start,homenr);
    }
    sf_dir = 0;
    if (nflexcon) {
      init_adir(fplog,shfc,
		constr,idef,inputrec,cr,dd_ac1,mdstep,md,start,end,
		x_old-start,state->x,pos[Try],force[Try],acc_dir-start,
		fr->bMolPBC,state->box,state->lambda,&dum,nrnb);

      for(i=start; i<end; i++)
	sf_dir += md->massT[i]*norm2(acc_dir[i-start]);
    }

    Epot[Try] = enerd->term[F_EPOT]; 
    
    df[Try]=rms_force(cr,force[Try],nshell,shell,nflexcon,&sf_dir,&Epot[Try]);

    if (debug)
      fprintf(debug,"df = %g  %g\n",df[Min],df[Try]);

    if (debug) {
      if (gmx_debug_at)
	pr_rvecs(debug,0,"F na do_force",force[Try] + start,homenr);
      if (gmx_debug_at) {
	fprintf(debug,"SHELL ITER %d\n",count);
	dump_shells(debug,pos[Try],force[Try],ftol,nshell,shell);
      }
    }

    if (bVerbose && MASTER(cr))
      print_epot(stdout,mdstep,count,Epot[Try],df[Try],nflexcon,sf_dir);
      
    *bConverged = (df[Try] < ftol);
    
    if ((df[Try] < df[Min])) {
      if (debug)
	fprintf(debug,"Swapping Min and Try\n");
      if (nflexcon) {
	/* Correct the velocities for the flexible constraints */
	invdt = 1/inputrec->delta_t;
	for(i=start; i<end; i++) {
	  for(d=0; d<DIM; d++)
	    state->v[i][d] += (pos[Try][i][d] - pos[Min][i][d])*invdt;
	}
      }
      Min  = Try;
    } else {
      decrease_step_size(nshell,shell);
    }
  }
  if (MASTER(cr) && !(*bConverged)) {
    /* Note that the energies and virial are incorrect when not converged */
    if (fplog)
      fprintf(fplog,
	      "step %s: EM did not converge in %d iterations, RMS force %.3f\n",
	      gmx_step_str(mdstep,sbuf),number_steps,df[Min]);
    fprintf(stderr,
	    "step %s: EM did not converge in %d iterations, RMS force %.3f\n",
	    gmx_step_str(mdstep,sbuf),number_steps,df[Min]);
  }

  /* Copy back the coordinates and the forces */
  memcpy(state->x,pos[Min],nat*sizeof(state->x[0]));
  memcpy(f,force[Min],nat*sizeof(f[0]));

  return count; 
}
Пример #2
0
void do_force(FILE *fplog,t_commrec *cr,
	      t_inputrec *inputrec,
	      int step,t_nrnb *nrnb,gmx_wallcycle_t wcycle,
	      gmx_localtop_t *top,
	      gmx_groups_t *groups,
	      matrix box,rvec x[],history_t *hist,
	      rvec f[],rvec buf[],
	      tensor vir_force,
	      t_mdatoms *mdatoms,
	      gmx_enerdata_t *enerd,t_fcdata *fcd,
	      real lambda,t_graph *graph,
	      t_forcerec *fr,gmx_vsite_t *vsite,rvec mu_tot,
	      real t,FILE *field,gmx_edsam_t ed,
	      int flags)
{
  static rvec box_size;
  int    cg0,cg1,i,j;
  int    start,homenr;
  static double mu[2*DIM]; 
  rvec   mu_tot_AB[2];
  bool   bSepDVDL,bStateChanged,bNS,bFillGrid,bCalcCGCM,bBS,bDoForces;
  matrix boxs;
  real   e,v,dvdl;
  t_pbc  pbc;
  float  cycles_ppdpme,cycles_pme,cycles_force;
  
  start  = mdatoms->start;
  homenr = mdatoms->homenr;

  bSepDVDL = (fr->bSepDVDL && do_per_step(step,inputrec->nstlog));

  clear_mat(vir_force);
  
  if (PARTDECOMP(cr)) {
    pd_cg_range(cr,&cg0,&cg1);
  } else {
    cg0 = 0;
    if (DOMAINDECOMP(cr))
      cg1 = cr->dd->ncg_tot;
    else
      cg1 = top->cgs.nr;
    if (fr->n_tpi > 0)
      cg1--;
  }

  bStateChanged = (flags & GMX_FORCE_STATECHANGED);
  bNS           = (flags & GMX_FORCE_NS);
  bFillGrid     = (bNS && bStateChanged);
  bCalcCGCM     = (bFillGrid && !DOMAINDECOMP(cr));
  bDoForces     = (flags & GMX_FORCE_FORCES);

  if (bStateChanged) {
    update_forcerec(fplog,fr,box);
    
    /* Calculate total (local) dipole moment in a temporary common array. 
     * This makes it possible to sum them over nodes faster.
     */
    calc_mu(start,homenr,
	    x,mdatoms->chargeA,mdatoms->chargeB,mdatoms->nChargePerturbed,
	    mu,mu+DIM);
  }
  
  if (fr->ePBC != epbcNONE) { 
    /* Compute shift vectors every step,
     * because of pressure coupling or box deformation!
     */
    if (DYNAMIC_BOX(*inputrec) && bStateChanged)
      calc_shifts(box,fr->shift_vec);
    
    if (bCalcCGCM) { 
      put_charge_groups_in_box(fplog,cg0,cg1,fr->ePBC,box,
			       &(top->cgs),x,fr->cg_cm);
      inc_nrnb(nrnb,eNR_CGCM,homenr);
      inc_nrnb(nrnb,eNR_RESETX,cg1-cg0);
    } 
    else if (EI_ENERGY_MINIMIZATION(inputrec->eI) && graph) {
      unshift_self(graph,box,x);
    }
  } 
  else if (bCalcCGCM) {
    calc_cgcm(fplog,cg0,cg1,&(top->cgs),x,fr->cg_cm);
    inc_nrnb(nrnb,eNR_CGCM,homenr);
  }
  
  if (bCalcCGCM) {
    if (PAR(cr)) {
      move_cgcm(fplog,cr,fr->cg_cm);
    }
    if (gmx_debug_at)
      pr_rvecs(debug,0,"cgcm",fr->cg_cm,top->cgs.nr);
  }

#ifdef GMX_MPI
  if (!(cr->duty & DUTY_PME)) {
    /* Send particle coordinates to the pme nodes.
     * Since this is only implemented for domain decomposition
     * and domain decomposition does not use the graph,
     * we do not need to worry about shifting.
     */    

    wallcycle_start(wcycle,ewcPP_PMESENDX);
    GMX_MPE_LOG(ev_send_coordinates_start);

    bBS = (inputrec->nwall == 2);
    if (bBS) {
      copy_mat(box,boxs);
      svmul(inputrec->wall_ewald_zfac,boxs[ZZ],boxs[ZZ]);
    }

    gmx_pme_send_x(cr,bBS ? boxs : box,x,mdatoms->nChargePerturbed,lambda);

    GMX_MPE_LOG(ev_send_coordinates_finish);
    wallcycle_stop(wcycle,ewcPP_PMESENDX);
  }
#endif /* GMX_MPI */

  /* Communicate coordinates and sum dipole if necessary */
  if (PAR(cr)) {
    wallcycle_start(wcycle,ewcMOVEX);
    if (DOMAINDECOMP(cr)) {
      dd_move_x(cr->dd,box,x,buf);
    } else {
      move_x(fplog,cr,GMX_LEFT,GMX_RIGHT,x,nrnb);
    }
    /* When we don't need the total dipole we sum it in global_stat */
    if (NEED_MUTOT(*inputrec))
      gmx_sumd(2*DIM,mu,cr);
    wallcycle_stop(wcycle,ewcMOVEX);
  }
  for(i=0; i<2; i++)
    for(j=0;j<DIM;j++)
      mu_tot_AB[i][j] = mu[i*DIM + j];
  if (fr->efep == efepNO)
    copy_rvec(mu_tot_AB[0],mu_tot);
  else
    for(j=0; j<DIM; j++)
      mu_tot[j] = (1.0 - lambda)*mu_tot_AB[0][j] + lambda*mu_tot_AB[1][j];

  /* Reset energies */
  reset_energies(&(inputrec->opts),fr,bNS,enerd,MASTER(cr));    
  if (bNS) {
    wallcycle_start(wcycle,ewcNS);
    
    if (graph && bStateChanged)
      /* Calculate intramolecular shift vectors to make molecules whole */
      mk_mshift(fplog,graph,fr->ePBC,box,x);

    /* Reset long range forces if necessary */
    if (fr->bTwinRange) {
      clear_rvecs(fr->f_twin_n,fr->f_twin);
      clear_rvecs(SHIFTS,fr->fshift_twin);
    }
    /* Do the actual neighbour searching and if twin range electrostatics
     * also do the calculation of long range forces and energies.
     */
    dvdl = 0; 
    ns(fplog,fr,x,f,box,groups,&(inputrec->opts),top,mdatoms,
       cr,nrnb,step,lambda,&dvdl,&enerd->grpp,bFillGrid,bDoForces);
    if (bSepDVDL)
      fprintf(fplog,sepdvdlformat,"LR non-bonded",0,dvdl);
    enerd->dvdl_lr       = dvdl;
    enerd->term[F_DVDL] += dvdl;

    wallcycle_stop(wcycle,ewcNS);
  }
  
  if (DOMAINDECOMP(cr)) {
    if (!(cr->duty & DUTY_PME)) {
      wallcycle_start(wcycle,ewcPPDURINGPME);
      dd_force_flop_start(cr->dd,nrnb);
    }
  }
  /* Start the force cycle counter.
   * This counter is stopped in do_forcelow_level.
   * No parallel communication should occur while this counter is running,
   * since that will interfere with the dynamic load balancing.
   */
  wallcycle_start(wcycle,ewcFORCE);

  if (bDoForces) {
      /* Reset PME/Ewald forces if necessary */
    if (fr->bF_NoVirSum) 
    {
      GMX_BARRIER(cr->mpi_comm_mygroup);
      if (fr->bDomDec)
	clear_rvecs(fr->f_novirsum_n,fr->f_novirsum);
      else
	clear_rvecs(homenr,fr->f_novirsum+start);
      GMX_BARRIER(cr->mpi_comm_mygroup);
    }
    /* Copy long range forces into normal buffers */
    if (fr->bTwinRange) {
      for(i=0; i<fr->f_twin_n; i++)
	copy_rvec(fr->f_twin[i],f[i]);
      for(i=0; i<SHIFTS; i++)
	copy_rvec(fr->fshift_twin[i],fr->fshift[i]);
    } 
    else {
      if (DOMAINDECOMP(cr))
	clear_rvecs(cr->dd->nat_tot,f);
      else
	clear_rvecs(mdatoms->nr,f);
      clear_rvecs(SHIFTS,fr->fshift);
    }
    clear_rvec(fr->vir_diag_posres);
    GMX_BARRIER(cr->mpi_comm_mygroup);
  }
  if (inputrec->ePull == epullCONSTRAINT)
    clear_pull_forces(inputrec->pull);

  /* update QMMMrec, if necessary */
  if(fr->bQMMM)
    update_QMMMrec(cr,fr,x,mdatoms,box,top);

  if ((flags & GMX_FORCE_BONDED) && top->idef.il[F_POSRES].nr > 0) {
    /* Position restraints always require full pbc */
    set_pbc(&pbc,inputrec->ePBC,box);
    v = posres(top->idef.il[F_POSRES].nr,top->idef.il[F_POSRES].iatoms,
	       top->idef.iparams_posres,
	       (const rvec*)x,fr->f_novirsum,fr->vir_diag_posres,
	       inputrec->ePBC==epbcNONE ? NULL : &pbc,lambda,&dvdl,
	       fr->rc_scaling,fr->ePBC,fr->posres_com,fr->posres_comB);
    if (bSepDVDL) {
      fprintf(fplog,sepdvdlformat,
	      interaction_function[F_POSRES].longname,v,dvdl);
    }
    enerd->term[F_POSRES] += v;
    enerd->term[F_DVDL]   += dvdl;
    inc_nrnb(nrnb,eNR_POSRES,top->idef.il[F_POSRES].nr/2);
  }
  /* Compute the bonded and non-bonded forces */    
  do_force_lowlevel(fplog,step,fr,inputrec,&(top->idef),
		    cr,nrnb,wcycle,mdatoms,&(inputrec->opts),
		    x,hist,f,enerd,fcd,box,lambda,graph,&(top->excls),mu_tot_AB,
		    flags,&cycles_force);
  GMX_BARRIER(cr->mpi_comm_mygroup);

  if (ed) {
    do_flood(fplog,cr,x,f,ed,box,step);
  }
	
  if (DOMAINDECOMP(cr)) {
    dd_force_flop_stop(cr->dd,nrnb);
    if (wcycle)
      dd_cycles_add(cr->dd,cycles_force,ddCyclF);
  }
  
  if (bDoForces) {
    /* Compute forces due to electric field */
    calc_f_el(MASTER(cr) ? field : NULL,
	      start,homenr,mdatoms->chargeA,x,f,inputrec->ex,inputrec->et,t);
    
    /* When using PME/Ewald we compute the long range virial there.
     * otherwise we do it based on long range forces from twin range
     * cut-off based calculation (or not at all).
     */
    
    /* Communicate the forces */
    if (PAR(cr)) {
      wallcycle_start(wcycle,ewcMOVEF);
      if (DOMAINDECOMP(cr)) {
	dd_move_f(cr->dd,f,buf,fr->fshift);
	/* Position restraint do not introduce inter-cg forces */
	if (EEL_FULL(fr->eeltype) && cr->dd->n_intercg_excl)
	  dd_move_f(cr->dd,fr->f_novirsum,buf,NULL);
      } else {
	move_f(fplog,cr,GMX_LEFT,GMX_RIGHT,f,buf,nrnb);
      }
      wallcycle_stop(wcycle,ewcMOVEF);
    }
  }

  if (bDoForces) {
    if (vsite) {
      wallcycle_start(wcycle,ewcVSITESPREAD);
      spread_vsite_f(fplog,vsite,x,f,fr->fshift,nrnb,
		     &top->idef,fr->ePBC,fr->bMolPBC,graph,box,cr);
      wallcycle_stop(wcycle,ewcVSITESPREAD);
    }
    
    /* Calculation of the virial must be done after vsites! */
    calc_virial(fplog,mdatoms->start,mdatoms->homenr,x,f,
		vir_force,graph,box,nrnb,fr,inputrec->ePBC);
  }

  if (inputrec->ePull == epullUMBRELLA || inputrec->ePull == epullCONST_F) {
    /* Calculate the center of mass forces, this requires communication,
     * which is why pull_potential is called close to other communication.
     * The virial contribution is calculated directly,
     * which is why we call pull_potential after calc_virial.
     */
    set_pbc(&pbc,inputrec->ePBC,box);
    dvdl = 0; 
    enerd->term[F_COM_PULL] =
      pull_potential(inputrec->ePull,inputrec->pull,mdatoms,&pbc,
		     cr,t,lambda,x,f,vir_force,&dvdl);
    if (bSepDVDL)
      fprintf(fplog,sepdvdlformat,"Com pull",enerd->term[F_COM_PULL],dvdl);
    enerd->term[F_DVDL] += dvdl;
  }

  if (!(cr->duty & DUTY_PME)) {
    cycles_ppdpme = wallcycle_stop(wcycle,ewcPPDURINGPME);
    dd_cycles_add(cr->dd,cycles_ppdpme,ddCyclPPduringPME);
  }

#ifdef GMX_MPI
  if (PAR(cr) && !(cr->duty & DUTY_PME)) {
    /* In case of node-splitting, the PP nodes receive the long-range 
     * forces, virial and energy from the PME nodes here.
     */    
    wallcycle_start(wcycle,ewcPP_PMEWAITRECVF);
    dvdl = 0;
    gmx_pme_receive_f(cr,fr->f_novirsum,fr->vir_el_recip,&e,&dvdl,
		      &cycles_pme);
    if (bSepDVDL)
      fprintf(fplog,sepdvdlformat,"PME mesh",e,dvdl);
    enerd->term[F_COUL_RECIP] += e;
    enerd->term[F_DVDL] += dvdl;
    if (wcycle)
      dd_cycles_add(cr->dd,cycles_pme,ddCyclPME);
    wallcycle_stop(wcycle,ewcPP_PMEWAITRECVF);
  }
#endif

  if (bDoForces && fr->bF_NoVirSum) {
    if (vsite) {
      /* Spread the mesh force on virtual sites to the other particles... 
       * This is parallellized. MPI communication is performed
       * if the constructing atoms aren't local.
       */
      wallcycle_start(wcycle,ewcVSITESPREAD);
      spread_vsite_f(fplog,vsite,x,fr->f_novirsum,NULL,nrnb,
		     &top->idef,fr->ePBC,fr->bMolPBC,graph,box,cr);
      wallcycle_stop(wcycle,ewcVSITESPREAD);
    }
    /* Now add the forces, this is local */
    if (fr->bDomDec) {
      sum_forces(0,fr->f_novirsum_n,f,fr->f_novirsum);
    } else {
      sum_forces(start,start+homenr,f,fr->f_novirsum);
    }
    if (EEL_FULL(fr->eeltype)) {
      /* Add the mesh contribution to the virial */
      m_add(vir_force,fr->vir_el_recip,vir_force);
    }
    if (debug)
      pr_rvecs(debug,0,"vir_force",vir_force,DIM);
  }

  /* Sum the potential energy terms from group contributions */
  sum_epot(&(inputrec->opts),enerd);

  if (fr->print_force >= 0 && bDoForces)
    print_large_forces(stderr,mdatoms,cr,step,fr->print_force,x,f);
}