void integrate_vv(int n_steps) { int i; /* Prepare the Integrator */ on_integration_start(); /* if any method vetoes (P3M not initialized), immediately bail out */ if (check_runtime_errors()) return; /* Verlet list criterion */ skin2 = SQR(0.5 * skin); INTEG_TRACE(fprintf(stderr,"%d: integrate_vv: integrating %d steps (recalc_forces=%d)\n", this_node, n_steps, recalc_forces)); /* Integration Step: Preparation for first integration step: Calculate forces f(t) as function of positions p(t) ( and velocities v(t) ) */ if (recalc_forces) { thermo_heat_up(); #ifdef LB transfer_momentum = 0; #endif #ifdef LB_GPU transfer_momentum_gpu = 0; #endif //VIRTUAL_SITES pos (and vel for DPD) update for security reason !!! #ifdef VIRTUAL_SITES update_mol_vel_pos(); ghost_communicator(&cell_structure.update_ghost_pos_comm); if (check_runtime_errors()) return; #ifdef ADRESS // adress_update_weights(); if (check_runtime_errors()) return; #endif #endif force_calc(); //VIRTUAL_SITES distribute forces #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.collect_ghost_force_comm); init_forces_ghosts(); distribute_mol_force(); if (check_runtime_errors()) return; #endif ghost_communicator(&cell_structure.collect_ghost_force_comm); #ifdef ROTATION convert_initial_torques(); #endif thermo_cool_down(); /* Communication Step: ghost forces */ /*apply trap forces to trapped molecules*/ #ifdef MOLFORCES calc_and_apply_mol_constraints(); #endif rescale_forces(); recalc_forces = 0; } if (check_runtime_errors()) return; n_verlet_updates = 0; /* Integration loop */ for(i=0;i<n_steps;i++) { INTEG_TRACE(fprintf(stderr,"%d: STEP %d\n",this_node,i)); #ifdef BOND_CONSTRAINT save_old_pos(); #endif /* Integration Steps: Step 1 and 2 of Velocity Verlet scheme: v(t+0.5*dt) = v(t) + 0.5*dt * f(t) p(t + dt) = p(t) + dt * v(t+0.5*dt) NOTE 1: Prefactors do not occur in formulas since we use rescaled forces and velocities. NOTE 2: Depending on the integration method Step 1 and Step 2 cannot be combined for the translation. */ if(integ_switch == INTEG_METHOD_NPT_ISO || nemd_method != NEMD_METHOD_OFF) { propagate_vel(); propagate_pos(); } else propagate_vel_pos(); #ifdef ROTATION propagate_omega_quat(); #endif #ifdef BOND_CONSTRAINT /**Correct those particle positions that participate in a rigid/constrained bond */ ghost_communicator(&cell_structure.update_ghost_pos_comm); correct_pos_shake(); #endif #ifdef ELECTROSTATICS if(coulomb.method == COULOMB_MAGGS) { maggs_propagate_B_field(0.5*time_step); } #endif #ifdef NPT if (check_runtime_errors()) break; #endif cells_update_ghosts(); //VIRTUAL_SITES update pos and vel (for DPD) #ifdef VIRTUAL_SITES update_mol_vel_pos(); ghost_communicator(&cell_structure.update_ghost_pos_comm); if (check_runtime_errors()) break; #if defined(VIRTUAL_SITES_RELATIVE) && defined(LB) // This is on a workaround stage: // When using virtual sites relative and LB at the same time, it is necessary // to reassemble the cell lists after all position updates, also of virtual // particles. if (cell_structure.type == CELL_STRUCTURE_DOMDEC && (!dd.use_vList) ) cells_update_ghosts(); #endif #ifdef ADRESS //adress_update_weights(); if (check_runtime_errors()) break; #endif #endif /* Integration Step: Step 3 of Velocity Verlet scheme: Calculate f(t+dt) as function of positions p(t+dt) ( and velocities v(t+0.5*dt) ) */ #ifdef LB transfer_momentum = 1; #endif #ifdef LB_GPU transfer_momentum_gpu = 1; #endif force_calc(); //VIRTUAL_SITES distribute forces #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.collect_ghost_force_comm); init_forces_ghosts(); distribute_mol_force(); if (check_runtime_errors()) break; #endif /* Communication step: ghost forces */ ghost_communicator(&cell_structure.collect_ghost_force_comm); /*apply trap forces to trapped molecules*/ #ifdef MOLFORCES calc_and_apply_mol_constraints(); #endif if (check_runtime_errors()) break; /* Integration Step: Step 4 of Velocity Verlet scheme: v(t+dt) = v(t+0.5*dt) + 0.5*dt * f(t+dt) */ rescale_forces_propagate_vel(); recalc_forces = 0; #ifdef LB if (lattice_switch & LATTICE_LB) lattice_boltzmann_update(); if (check_runtime_errors()) break; #endif #ifdef LB_GPU if(this_node == 0){ if (lattice_switch & LATTICE_LB_GPU) lattice_boltzmann_update_gpu(); } #endif #ifdef BOND_CONSTRAINT ghost_communicator(&cell_structure.update_ghost_pos_comm); correct_vel_shake(); #endif #ifdef ROTATION convert_torques_propagate_omega(); #endif //VIRTUAL_SITES update vel #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.update_ghost_pos_comm); update_mol_vel(); if (check_runtime_errors()) break; #endif #ifdef ELECTROSTATICS if(coulomb.method == COULOMB_MAGGS) { maggs_propagate_B_field(0.5*time_step); } #endif #ifdef NPT if((this_node==0) && (integ_switch == INTEG_METHOD_NPT_ISO)) nptiso.p_inst_av += nptiso.p_inst; #endif /* Propagate time: t = t+dt */ sim_time += time_step; } /* verlet list statistics */ if(n_verlet_updates>0) verlet_reuse = n_steps/(double) n_verlet_updates; else verlet_reuse = 0; #ifdef NPT if(integ_switch == INTEG_METHOD_NPT_ISO) { nptiso.invalidate_p_vel = 0; MPI_Bcast(&nptiso.p_inst, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Bcast(&nptiso.p_diff, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Bcast(&nptiso.volume, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); if(this_node==0) nptiso.p_inst_av /= 1.0*n_steps; MPI_Bcast(&nptiso.p_inst_av, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); } #endif }
void force_calc() { // Communication step: distribute ghost positions cells_update_ghosts(); // VIRTUAL_SITES pos (and vel for DPD) update for security reason !!! #ifdef VIRTUAL_SITES update_mol_vel_pos(); ghost_communicator(&cell_structure.update_ghost_pos_comm); #endif #if defined(VIRTUAL_SITES_RELATIVE) && defined(LB) // This is on a workaround stage: // When using virtual sites relative and LB at the same time, it is necessary // to reassemble the cell lists after all position updates, also of virtual // particles. if ((lattice_switch & LATTICE_LB) && cell_structure.type == CELL_STRUCTURE_DOMDEC && (!dd.use_vList) ) cells_update_ghosts(); #endif espressoSystemInterface.update(); #ifdef COLLISION_DETECTION prepare_collision_queue(); #endif #ifdef LB_GPU #ifdef SHANCHEN if (lattice_switch & LATTICE_LB_GPU && this_node == 0) lattice_boltzmann_calc_shanchen_gpu(); #endif // SHANCHEN // transfer_momentum_gpu check makes sure the LB fluid doesn't get updated on integrate 0 // this_node==0 makes sure it is the master node where the gpu exists if (lattice_switch & LATTICE_LB_GPU && transfer_momentum_gpu && (this_node == 0) ) lb_calc_particle_lattice_ia_gpu(); #endif // LB_GPU #ifdef ELECTROSTATICS if (iccp3m_initialized && iccp3m_cfg.set_flag) iccp3m_iteration(); #endif init_forces(); for (ActorList::iterator actor = forceActors.begin(); actor != forceActors.end(); ++actor) { (*actor)->computeForces(espressoSystemInterface); #ifdef ROTATION (*actor)->computeTorques(espressoSystemInterface); #endif } calc_long_range_forces(); switch (cell_structure.type) { case CELL_STRUCTURE_LAYERED: layered_calculate_ia(); break; case CELL_STRUCTURE_DOMDEC: if(dd.use_vList) { if (rebuild_verletlist) build_verlet_lists_and_calc_verlet_ia(); else calculate_verlet_ia(); } else calc_link_cell(); break; case CELL_STRUCTURE_NSQUARE: nsq_calculate_ia(); } #ifdef OIF_GLOBAL_FORCES double area_volume[2]; //There are two global quantities that need to be evaluated: object's surface and object's volume. One can add another quantity. area_volume[0] = 0.0; area_volume[1] = 0.0; for (int i=0;i< MAX_OBJECTS_IN_FLUID;i++){ calc_oif_global(area_volume,i); if (fabs(area_volume[0])<1e-100 && fabs(area_volume[1])<1e-100) break; add_oif_global_forces(area_volume,i); } #endif #ifdef IMMERSED_BOUNDARY // Must be done here. Forces need to be ghost-communicated IBM_VolumeConservation(); #endif #ifdef LB if (lattice_switch & LATTICE_LB) calc_particle_lattice_ia() ; #endif #ifdef COMFORCE calc_comforce(); #endif #ifdef METADYNAMICS /* Metadynamics main function */ meta_perform(); #endif #ifdef CUDA copy_forces_from_GPU(); #endif // VIRTUAL_SITES distribute forces #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.collect_ghost_force_comm); init_forces_ghosts(); distribute_mol_force(); #endif // Communication Step: ghost forces ghost_communicator(&cell_structure.collect_ghost_force_comm); // apply trap forces to trapped molecules #ifdef MOLFORCES calc_and_apply_mol_constraints(); #endif // should be pretty late, since it needs to zero out the total force #ifdef COMFIXED calc_comfixed(); #endif // mark that forces are now up-to-date recalc_forces = 0; }
void integrate_vv(int n_steps) { int i; /* Prepare the Integrator */ on_integration_start(); /* if any method vetoes (P3M not initialized), immediately bail out */ if (check_runtime_errors()) return; /* Verlet list criterion */ skin2 = SQR(0.5 * skin); INTEG_TRACE(fprintf(stderr,"%d: integrate_vv: integrating %d steps (recalc_forces=%d)\n", this_node, n_steps, recalc_forces)); /* Integration Step: Preparation for first integration step: Calculate forces f(t) as function of positions p(t) ( and velocities v(t) ) */ if (recalc_forces) { thermo_heat_up(); #ifdef LB transfer_momentum = 0; #endif #ifdef LB_GPU transfer_momentum_gpu = 0; #endif //VIRTUAL_SITES pos (and vel for DPD) update for security reason !!! #ifdef VIRTUAL_SITES update_mol_vel_pos(); ghost_communicator(&cell_structure.update_ghost_pos_comm); if (check_runtime_errors()) return; #ifdef ADRESS // adress_update_weights(); if (check_runtime_errors()) return; #endif #endif force_calc(); //VIRTUAL_SITES distribute forces #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.collect_ghost_force_comm); init_forces_ghosts(); distribute_mol_force(); if (check_runtime_errors()) return; #endif ghost_communicator(&cell_structure.collect_ghost_force_comm); #ifdef ROTATION convert_initial_torques(); #endif thermo_cool_down(); /* Communication Step: ghost forces */ /*apply trap forces to trapped molecules*/ #ifdef MOLFORCES calc_and_apply_mol_constraints(); #endif rescale_forces(); recalc_forces = 0; } if (check_runtime_errors()) return; n_verlet_updates = 0; /* Integration loop */ for(i=0;i<n_steps;i++) { INTEG_TRACE(fprintf(stderr,"%d: STEP %d\n",this_node,i)); #ifdef BOND_CONSTRAINT save_old_pos(); #endif /* Integration Steps: Step 1 and 2 of Velocity Verlet scheme: v(t+0.5*dt) = v(t) + 0.5*dt * f(t) p(t + dt) = p(t) + dt * v(t+0.5*dt) NOTE 1: Prefactors do not occur in formulas since we use rescaled forces and velocities. NOTE 2: Depending on the integration method Step 1 and Step 2 cannot be combined for the translation. */ if(integ_switch == INTEG_METHOD_NPT_ISO || nemd_method != NEMD_METHOD_OFF) { propagate_vel(); propagate_pos(); } else propagate_vel_pos(); #ifdef ROTATION propagate_omega_quat(); #endif #ifdef BOND_CONSTRAINT /**Correct those particle positions that participate in a rigid/constrained bond */ ghost_communicator(&cell_structure.update_ghost_pos_comm); correct_pos_shake(); #endif #ifdef ELECTROSTATICS if(coulomb.method == COULOMB_MAGGS) { propagate_B_field(0.5*time_step); if(maggs.yukawa) maggs_propagate_psi_vel_pos(time_step); } #endif #ifdef NPT if (check_runtime_errors()) break; #endif cells_update_ghosts(); //VIRTUAL_SITES update pos and vel (for DPD) #ifdef VIRTUAL_SITES update_mol_vel_pos(); ghost_communicator(&cell_structure.update_ghost_pos_comm); if (check_runtime_errors()) break; #ifdef ADRESS //adress_update_weights(); if (check_runtime_errors()) break; #endif #endif /* Integration Step: Step 3 of Velocity Verlet scheme: Calculate f(t+dt) as function of positions p(t+dt) ( and velocities v(t+0.5*dt) ) */ #ifdef LB transfer_momentum = 1; #endif #ifdef LB_GPU transfer_momentum_gpu = 1; #endif force_calc(); //VIRTUAL_SITES distribute forces #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.collect_ghost_force_comm); init_forces_ghosts(); distribute_mol_force(); if (check_runtime_errors()) break; #endif /* Communication step: ghost forces */ ghost_communicator(&cell_structure.collect_ghost_force_comm); /*apply trap forces to trapped molecules*/ #ifdef MOLFORCES calc_and_apply_mol_constraints(); #endif if (check_runtime_errors()) break; /* Integration Step: Step 4 of Velocity Verlet scheme: v(t+dt) = v(t+0.5*dt) + 0.5*dt * f(t+dt) */ rescale_forces_propagate_vel(); #ifdef LB if (lattice_switch & LATTICE_LB) lattice_boltzmann_update(); if (check_runtime_errors()) break; #endif #ifdef LB_GPU if(this_node == 0){ if (lattice_switch & LATTICE_LB_GPU) lattice_boltzmann_update_gpu(); } #endif #ifdef BOND_CONSTRAINT ghost_communicator(&cell_structure.update_ghost_pos_comm); correct_vel_shake(); #endif //VIRTUAL_SITES update vel #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.update_ghost_pos_comm); update_mol_vel(); if (check_runtime_errors()) break; #endif #ifdef ELECTROSTATICS if(coulomb.method == COULOMB_MAGGS) { propagate_B_field(0.5*time_step); if(maggs.yukawa) maggs_propagate_psi_vel(0.5*time_step); MAGGS_TRACE(check_gauss_law();); }
void integrate_vv(int n_steps) { int i; #ifdef REACTIONS int c, np, n; Particle * p1, *p2, **pairs; Cell * cell; double dist2, vec21[3], rand; if(reaction.rate > 0) { int reactants = 0, products = 0; int tot_reactants = 0, tot_products = 0; double ratexp, back_ratexp; ratexp = exp(-time_step*reaction.rate); on_observable_calc(); for (c = 0; c < local_cells.n; c++) { /* Loop cell neighbors */ for (n = 0; n < dd.cell_inter[c].n_neighbors; n++) { pairs = dd.cell_inter[c].nList[n].vList.pair; np = dd.cell_inter[c].nList[n].vList.n; /* verlet list loop */ for(i = 0; i < 2 * np; i += 2) { p1 = pairs[i]; //pointer to particle 1 p2 = pairs[i+1]; //pointer to particle 2 if( (p1->p.type == reaction.reactant_type && p2->p.type == reaction.catalyzer_type) || (p2->p.type == reaction.reactant_type && p1->p.type == reaction.catalyzer_type) ) { get_mi_vector(vec21, p1->r.p, p2->r.p); dist2 = sqrlen(vec21); if(dist2 < reaction.range * reaction.range) { rand =d_random(); if(rand > ratexp) { if(p1->p.type==reaction.reactant_type) { p1->p.type = reaction.product_type; } else { p2->p.type = reaction.product_type; } } } } } } } if (reaction.back_rate < 0) { // we have to determine it dynamically /* we count now how many reactants and products are in the sim box */ for (c = 0; c < local_cells.n; c++) { cell = local_cells.cell[c]; p1 = cell->part; np = cell->n; for(i = 0; i < np; i++) { if(p1[i].p.type == reaction.reactant_type) reactants++; else if(p1[i].p.type == reaction.product_type) products++; } } MPI_Allreduce(&reactants, &tot_reactants, 1, MPI_INT, MPI_SUM, comm_cart); MPI_Allreduce(&products, &tot_products, 1, MPI_INT, MPI_SUM, comm_cart); back_ratexp = ratexp * tot_reactants / tot_products ; //with this the asymptotic ratio reactant/product becomes 1/1 and the catalyzer volume only determines the time that it takes to reach this } else { //use the back reaction rate supplied by the user back_ratexp = exp(-time_step*reaction.back_rate); } if(back_ratexp < 1 ) { for (c = 0; c < local_cells.n; c++) { cell = local_cells.cell[c]; p1 = cell->part; np = cell->n; for(i = 0; i < np; i++) { if(p1[i].p.type == reaction.product_type) { rand = d_random(); if(rand > back_ratexp) { p1[i].p.type=reaction.reactant_type; } } } } } on_particle_change(); } #endif /* ifdef REACTIONS */ /* Prepare the Integrator */ on_integration_start(); /* if any method vetoes (P3M not initialized), immediately bail out */ if (check_runtime_errors()) return; /* Verlet list criterion */ skin2 = SQR(0.5 * skin); INTEG_TRACE(fprintf(stderr,"%d: integrate_vv: integrating %d steps (recalc_forces=%d)\n", this_node, n_steps, recalc_forces)); /* Integration Step: Preparation for first integration step: Calculate forces f(t) as function of positions p(t) ( and velocities v(t) ) */ if (recalc_forces) { thermo_heat_up(); #ifdef LB transfer_momentum = 0; #endif #ifdef LB_GPU transfer_momentum_gpu = 0; #endif //VIRTUAL_SITES pos (and vel for DPD) update for security reason !!! #ifdef VIRTUAL_SITES update_mol_vel_pos(); ghost_communicator(&cell_structure.update_ghost_pos_comm); if (check_runtime_errors()) return; #ifdef ADRESS // adress_update_weights(); if (check_runtime_errors()) return; #endif #endif #ifdef COLLISION_DETECTION prepare_collision_queue(); #endif force_calc(); //VIRTUAL_SITES distribute forces #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.collect_ghost_force_comm); init_forces_ghosts(); distribute_mol_force(); if (check_runtime_errors()) return; #endif ghost_communicator(&cell_structure.collect_ghost_force_comm); #ifdef ROTATION convert_initial_torques(); #endif thermo_cool_down(); /* Communication Step: ghost forces */ /*apply trap forces to trapped molecules*/ #ifdef MOLFORCES calc_and_apply_mol_constraints(); #endif rescale_forces(); recalc_forces = 0; #ifdef COLLISION_DETECTION handle_collisions(); #endif } if (check_runtime_errors()) return; n_verlet_updates = 0; /* Integration loop */ for(i=0;i<n_steps;i++) { INTEG_TRACE(fprintf(stderr,"%d: STEP %d\n",this_node,i)); #ifdef BOND_CONSTRAINT save_old_pos(); #endif /* Integration Steps: Step 1 and 2 of Velocity Verlet scheme: v(t+0.5*dt) = v(t) + 0.5*dt * f(t) p(t + dt) = p(t) + dt * v(t+0.5*dt) NOTE 1: Prefactors do not occur in formulas since we use rescaled forces and velocities. NOTE 2: Depending on the integration method Step 1 and Step 2 cannot be combined for the translation. */ if(integ_switch == INTEG_METHOD_NPT_ISO || nemd_method != NEMD_METHOD_OFF) { propagate_vel(); propagate_pos(); } else propagate_vel_pos(); #ifdef BOND_CONSTRAINT /**Correct those particle positions that participate in a rigid/constrained bond */ cells_update_ghosts(); correct_pos_shake(); #endif #ifdef ELECTROSTATICS if(coulomb.method == COULOMB_MAGGS) { maggs_propagate_B_field(0.5*time_step); } #endif #ifdef NPT if (check_runtime_errors()) break; #endif cells_update_ghosts(); //VIRTUAL_SITES update pos and vel (for DPD) #ifdef VIRTUAL_SITES update_mol_vel_pos(); ghost_communicator(&cell_structure.update_ghost_pos_comm); if (check_runtime_errors()) break; #if defined(VIRTUAL_SITES_RELATIVE) && defined(LB) // This is on a workaround stage: // When using virtual sites relative and LB at the same time, it is necessary // to reassemble the cell lists after all position updates, also of virtual // particles. if (cell_structure.type == CELL_STRUCTURE_DOMDEC && (!dd.use_vList) ) cells_update_ghosts(); #endif #ifdef ADRESS //adress_update_weights(); if (check_runtime_errors()) break; #endif #endif /* Integration Step: Step 3 of Velocity Verlet scheme: Calculate f(t+dt) as function of positions p(t+dt) ( and velocities v(t+0.5*dt) ) */ #ifdef LB transfer_momentum = 1; #endif #ifdef LB_GPU transfer_momentum_gpu = 1; #endif #ifdef COLLISION_DETECTION prepare_collision_queue(); #endif force_calc(); //VIRTUAL_SITES distribute forces #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.collect_ghost_force_comm); init_forces_ghosts(); distribute_mol_force(); if (check_runtime_errors()) break; #endif /* Communication step: ghost forces */ ghost_communicator(&cell_structure.collect_ghost_force_comm); /*apply trap forces to trapped molecules*/ #ifdef MOLFORCES calc_and_apply_mol_constraints(); #endif if (check_runtime_errors()) break; /* Integration Step: Step 4 of Velocity Verlet scheme: v(t+dt) = v(t+0.5*dt) + 0.5*dt * f(t+dt) */ rescale_forces_propagate_vel(); recalc_forces = 0; #ifdef LB if (lattice_switch & LATTICE_LB) lattice_boltzmann_update(); if (check_runtime_errors()) break; #endif #ifdef LB_GPU if(this_node == 0){ if (lattice_switch & LATTICE_LB_GPU) lattice_boltzmann_update_gpu(); } #endif #ifdef BOND_CONSTRAINT ghost_communicator(&cell_structure.update_ghost_pos_comm); correct_vel_shake(); #endif #ifdef ROTATION convert_torques_propagate_omega(); #endif //VIRTUAL_SITES update vel #ifdef VIRTUAL_SITES ghost_communicator(&cell_structure.update_ghost_pos_comm); update_mol_vel(); if (check_runtime_errors()) break; #endif #ifdef ELECTROSTATICS if(coulomb.method == COULOMB_MAGGS) { maggs_propagate_B_field(0.5*time_step); } #endif #ifdef COLLISION_DETECTION handle_collisions(); #endif #ifdef NPT if((this_node==0) && (integ_switch == INTEG_METHOD_NPT_ISO)) nptiso.p_inst_av += nptiso.p_inst; #endif /* Propagate time: t = t+dt */ sim_time += time_step; } /* verlet list statistics */ if(n_verlet_updates>0) verlet_reuse = n_steps/(double) n_verlet_updates; else verlet_reuse = 0; #ifdef NPT if(integ_switch == INTEG_METHOD_NPT_ISO) { nptiso.invalidate_p_vel = 0; MPI_Bcast(&nptiso.p_inst, 1, MPI_DOUBLE, 0, comm_cart); MPI_Bcast(&nptiso.p_diff, 1, MPI_DOUBLE, 0, comm_cart); MPI_Bcast(&nptiso.volume, 1, MPI_DOUBLE, 0, comm_cart); if(this_node==0) nptiso.p_inst_av /= 1.0*n_steps; MPI_Bcast(&nptiso.p_inst_av, 1, MPI_DOUBLE, 0, comm_cart); } #endif }