int test_solver(const char *method, MPI_Comm sub_comm)
{
int sub_size, sub_rank;
fcs_float offset[] = { 0.0, 0.0, 0.0 };
fcs_float box_a[] = { 1.0, 0.0, 0.0 };
fcs_float box_b[] = { 0.0, 1.0, 0.0 };
fcs_float box_c[] = { 0.0, 0.0, 1.0 };
fcs_int periodicity[] = { 1, 1, 1 };
fcs_int local_nparticles = 4;
fcs_float positions[local_nparticles * 3], charges[local_nparticles];
fcs_float field[local_nparticles * 3], potentials[local_nparticles];
FCS fcs;
FCSResult result;
MPI_Comm_size(sub_comm, &sub_size);
MPI_Comm_rank(sub_comm, &sub_rank);
srandom(sub_rank * 2501);
for (fcs_int i = 0; i < local_nparticles; ++i)
{
positions[3 * i + 0] = (fcs_float) random() / (fcs_float) RAND_MAX;
positions[3 * i + 1] = (fcs_float) random() / (fcs_float) RAND_MAX;
positions[3 * i + 2] = (fcs_float) random() / (fcs_float) RAND_MAX;
charges[i] = 1.0;
}
result = fcs_init(&fcs, method, sub_comm);
if (result != FCS_RESULT_SUCCESS)
{
if (fcsResult_getReturnCode(result) == FCS_WRONG_ARGUMENT)
{
printf("%d: solver method '%s' not available!\n", rank, method);
return 10;
}
printf("%d: fcs_init failed!\n", rank);
return 11;
}
result = fcs_set_common(fcs, 1, box_a, box_b, box_c, offset, periodicity, local_nparticles * sub_size);
if (result != FCS_RESULT_SUCCESS) { printf("%d: fcs_set_common failed!\n", rank); return 12; }
result = fcs_run(fcs, local_nparticles, local_nparticles, positions, charges, field, potentials);
if (result != FCS_RESULT_SUCCESS) { printf("%d: fcs_run failed!\n", rank); return 13; }
result = fcs_destroy(fcs);
if (result != FCS_RESULT_SUCCESS) { printf("%d: fcs_destroy failed!\n", rank); return 14; }
return 0;
}
int main (int argc, char **argv)
{
char **arguments = argv;
int arg_count = argc;
/* time values for measuring near field time */
/* double t1 = 0.0, t2 = 0.0, tmax = 0.0; */
char *filename;
fcs_int number_of_runs;
MPI_Comm communicator = MPI_COMM_WORLD;
fcs_float *box_a;
fcs_float *box_b;
fcs_float *box_c;
fcs_float *offset;
fcs_float *particles;
fcs_float *charges;
fcs_float *velocities;
fcs_float *masses;
fcs_float *field;
fcs_float *potentials;
fcs_int periodicity[3];
#if FCS_ENABLE_FMM
char FMM_parameters[] = "fmm_absrel,2,fmm_dipole_correction,0,fmm_tolerance_energy,1e-6,fmm_internal_tuning,0ll";
#endif
#if FCS_ENABLE_PP3MG
char PP3MG_parameters[] = "pp3mg_cells_x,128,pp3mg_cells_y,128,pp3mg_cells_z,128,pp3mg_ghosts,4";
#endif
#if FCS_ENABLE_P2NFFT
char P2NFFT_parameters[] = "P2NFFT_required_accuracy,1e-6";
#endif
FILE *input_file;
FILE *xyz_file;
fcs_int local_particles;
char line[400];
fcs_int number_of_particles;
fcs_int i, j, k;
int comm_rank, comm_size;
int dims[3], pers[3];
char c_dummy;
/*different handles for different methods*/
#if FCS_ENABLE_FMM
FCS handle_fmm = NULL;
#endif
#if FCS_ENABLE_MEMD
FCS handle_memd = NULL;
#endif
#if FCS_ENABLE_PEPC
FCS handle_pepc = NULL;
#endif
#if FCS_ENABLE_P2NFFT
FCS handle_p2nfft = NULL;
#endif
#if FCS_ENABLE_P3M
FCS handle_p3m = NULL;
#endif
#if FCS_ENABLE_PP3MG
FCS handle_pp3mg = NULL;
#endif
#if FCS_ENABLE_VMG
FCS handle_vmg = NULL;
#endif
FCSResult result = NULL;
const fcs_float time_step = 1e-7;
const fcs_float charge_scale = 1.602e-19;
const fcs_float mass_scale = 1.6e-26;
/* scaling factor for MD steps */
const fcs_float scaling_factor = time_step * charge_scale / mass_scale;
MPI_Comm cart_comm;
char common_parameters[] = "box_a,1.0,0.0,0.0,box_b,0.0,1.0,0.0,box_c,0.0,0.0,1.0,periodicity,1,1,1,offset,0.0,0.0,0.0,near_field_flag,1";
periodicity[0] = 1;
periodicity[1] = 1;
periodicity[2] = 1;
dims[0] = dims[1] = dims[2] = 0;
pers[0] = periodicity[0];
pers[1] = periodicity[1];
pers[2] = periodicity[2];
/* command line handling */
MPI_Init (&arg_count, &arguments);
MPI_Comm_size (communicator, &comm_size);
fprintf (stderr, "comm_size: %d \n", comm_size);
MPI_Dims_create (comm_size, 3, dims);
fprintf (stderr, "dims: %d %d %d\n", dims[0], dims[1], dims[2]);
MPI_Cart_create (communicator, 3, dims, pers, 0, &cart_comm);
communicator = cart_comm;
if (argc < 2 || argc > 3)
{
fprintf (stderr,
"Usage: %s <solver> <input_file_name> [<number_of_runs>]\n",
argv[0]);
fprintf (stderr,
" currently only the files in test_env/data are supported\n");
return (-1);
}
filename = argv[1];
number_of_runs = 1;
if (argc > 2)
number_of_runs = atoi (argv[2]);
MPI_Comm_rank (communicator, &comm_rank);
MPI_Comm_size (communicator, &comm_size);
if (comm_rank == 0)
printf ("Interface test\n");
input_file = fopen (filename, "r");
if (input_file == NULL)
{
fprintf (stderr, "ERROR: file %s could not be opened!\n", filename);
return 2;
}
for (i = 0; i < 4; ++i)
{
fgets (line, 400, input_file);
}
fscanf (input_file, "%c%" FCS_CONV_INT "d\n", &c_dummy, &number_of_particles);
/* dividing particles up */
if (comm_rank == comm_size - 1)
local_particles =
number_of_particles - (comm_size -
1) * (number_of_particles / comm_size);
else
local_particles = number_of_particles / comm_size;
particles = (fcs_float *) malloc (3 * sizeof (fcs_float) * local_particles);
charges = (fcs_float *) malloc (3 * sizeof (fcs_float) * local_particles);
velocities =
(fcs_float *) malloc (3 * sizeof (fcs_float) * local_particles);
masses = (fcs_float *) malloc (sizeof (fcs_float) * local_particles);
field = (fcs_float *) malloc (3 * sizeof (fcs_float) * local_particles);
potentials = (fcs_float *) malloc (sizeof (fcs_float) * local_particles);
/* printf("%d particles: %d/%d\n", comm_rank, local_particles, number_of_particles); */
j = 0;
for (i = 0; i < number_of_particles; ++i)
{
if (((i >= comm_rank * local_particles)
|| i >= number_of_particles - local_particles)
&& i < (comm_rank + 1) * local_particles)
{
fscanf (input_file,
"%" FCS_CONV_FLOAT "f\t %" FCS_CONV_FLOAT "f\t %" FCS_CONV_FLOAT "f\t %" FCS_CONV_FLOAT "f\t %" FCS_CONV_FLOAT "f\t %" FCS_CONV_FLOAT "f\t %" FCS_CONV_FLOAT "f\t %" FCS_CONV_FLOAT "f\n",
(particles + j * 3), (particles + j * 3 + 1),
(particles + j * 3 + 2), masses + j, (charges + j),
(velocities + j * 3), (velocities + j * 3 + 1),
(velocities + j * 3 + 2));
j++;
}
else
{
fgets (line, 400, input_file);
}
}
box_a = (fcs_float *) malloc (3 * sizeof (fcs_float));
box_b = (fcs_float *) malloc (3 * sizeof (fcs_float));
box_c = (fcs_float *) malloc (3 * sizeof (fcs_float));
offset = (fcs_float *) malloc (3 * sizeof (fcs_float));
box_a[0] = box_a[1] = box_a[2] = box_b[0] = box_b[1] = box_b[2] = box_c[0] =
box_c[1] = box_c[2] = 0.0;
box_a[0] = INTERFACE_TEST_BOX_SIZE;
box_b[1] = INTERFACE_TEST_BOX_SIZE;
box_c[2] = INTERFACE_TEST_BOX_SIZE;
offset[0] = 0.0;
offset[1] = 0.0;
offset[2] = 0.0;
if (comm_rank == 0)
printf ("\n");
#if FCS_ENABLE_PP3MG
if (comm_rank == 0)
printf ("Initializing %s method...\n", "pp3mg");
result = fcs_init (&handle_pp3mg, "pp3mg", communicator);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_FMM
if (comm_rank == 0)
printf ("Initializing %s method...\n", "fmm");
result = fcs_init (&handle_fmm, "fmm", communicator);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_MEMD
if (comm_rank == 0)
printf ("Initializing %s method...\n", "memd");
result = fcs_init (&handle_memd, "memd", communicator);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_PEPC
if (comm_rank == 0)
printf ("Initializing %s method...\n", "pepc");
result = fcs_init (&handle_pepc, "pepc", communicator);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_P2NFFT
if (comm_rank == 0)
printf ("Initializing %s method...\n", "p2nfft");
result = fcs_init (&handle_p2nfft, "p2nfft", communicator);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_P3M
if (comm_rank == 0)
printf ("Initializing %s method...\n", "p3m");
result = fcs_init (&handle_p3m, "p3m", communicator);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_VMG
if (comm_rank == 0)
printf ("Initializing %s method...\n", "vmg");
result = fcs_init (&handle_vmg, "vmg", communicator);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
MPI_Barrier(communicator);
if (comm_rank == 0)
printf ("\n");
MPI_Barrier(communicator);
#if FCS_ENABLE_PP3MG
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Setting common data for pp3mg...\n");
result = fcs_set_parameters(handle_pp3mg, common_parameters, FCS_FALSE);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
result = fcs_set_total_particles(handle_pp3mg, number_of_particles);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Parameters of pp3mg... \n");
if (comm_rank == 0)
fcs_print_parameters (handle_pp3mg);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Tuning pp3mg... \n");
result =
fcs_tune (handle_pp3mg, local_particles, particles, charges);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_MEMD
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Setting common data for memd...\n");
result = fcs_set_parameters(handle_memd, common_parameters, FCS_FALSE);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
result = fcs_set_total_particles(handle_memd, number_of_particles);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Parameters of memd... \n");
if (comm_rank == 0)
fcs_print_parameters (handle_memd);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Tuning memd... \n");
result =
fcs_tune (handle_memd, local_particles, particles, charges);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_PEPC
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Setting common data for pepc...\n");
result = fcs_set_parameters(handle_pepc, common_parameters, FCS_FALSE);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
result = fcs_set_total_particles(handle_pepc, number_of_particles);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Parameters of pepc... \n");
if (comm_rank == 0)
fcs_print_parameters (handle_pepc);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Tuning pepc... \n");
result =
fcs_tune (handle_pepc, local_particles, particles, charges);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_FMM
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Setting common data for fmm...\n");
result = fcs_set_parameters(handle_fmm, common_parameters, FCS_FALSE);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
result = fcs_set_total_particles(handle_fmm, number_of_particles);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Parameters of fmm... \n");
if (comm_rank == 0)
fcs_print_parameters (handle_fmm);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Tuning fmm... \n");
result =
fcs_tune (handle_fmm, local_particles, particles, charges);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_P2NFFT
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Setting common data for p2nfft...\n");
result = fcs_set_parameters(handle_p2nfft, common_parameters, FCS_FALSE);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
result = fcs_set_total_particles(handle_p2nfft, number_of_particles);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Parameters of p2nfft ... \n");
if (comm_rank == 0)
fcs_print_parameters (handle_p2nfft);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Tuning p2nfft... \n");
result =
fcs_tune (handle_p2nfft, local_particles, particles, charges);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_P3M
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Setting common data for p3m...\n");
result = fcs_set_parameters(handle_p3m, common_parameters, FCS_FALSE);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
result = fcs_set_total_particles(handle_p3m, number_of_particles);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Parameters of p3m ... \n");
if (comm_rank == 0)
fcs_print_parameters (handle_p3m);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Tuning p3m... \n");
result =
fcs_tune (handle_p3m, local_particles, particles, charges);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
#if FCS_ENABLE_VMG
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Setting common data for vmg...\n");
result = fcs_set_parameters(handle_vmg, common_parameters, FCS_FALSE);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
result = fcs_set_total_particles(handle_vmg, number_of_particles);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Parameters of vmg... \n");
if (comm_rank == 0)
fcs_print_parameters (handle_vmg);
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Tuning vmg... \n");
result =
fcs_tune (handle_vmg, local_particles, particles, charges);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
#endif
MPI_Barrier(communicator);
for (i = 0; i < number_of_runs; ++i)
{
switch(i%6)
{
#if FCS_ENABLE_PP3MG
case 0:
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Running pp3mg ... \n");
result = fcs_run (handle_pp3mg, local_particles, particles,
charges, field, potentials);
break;
#endif
#if FCS_ENABLE_FMM
case 1:
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Running fmm ... \n");
result = fcs_run (handle_fmm, local_particles, particles,
charges, field, potentials);
break;
#endif
#if FCS_ENABLE_MEMD
case -1:
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Running memd ... \n");
result = fcs_run (handle_memd, local_particles, particles,
charges, field, potentials);
break;
#endif
#if FCS_ENABLE_P2NFFT
case 2:
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Running p2nfft ... \n");
result = fcs_run (handle_p2nfft, local_particles, particles,
charges, field, potentials);
break;
#endif
#if FCS_ENABLE_P3M
case 3:
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Running p3m ... \n");
result = fcs_run (handle_p3m, local_particles, particles,
charges, field, potentials);
break;
#endif
#if FCS_ENABLE_PEPC
case 4:
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Running pepc ... \n");
result = fcs_run (handle_pepc, local_particles, particles,
charges, field, potentials);
break;
#endif
#if FCS_ENABLE_VMG
case 5:
if (comm_rank == 0)
printf ("\n");
if (comm_rank == 0)
printf ("Running vmg ... \n");
result = fcs_run (handle_vmg, local_particles, particles,
charges, field, potentials);
break;
#endif
default:
printf( "Skipping step due to missing method \n");
continue;
break;
}
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
// if(comm_rank == 0) printf("Moving particles: %d... \n", i+1);
// for (j = 0; j < local_particles; ++j)
// {
// for (k = 0; k < 3; ++k)
// {
// *(velocities + j * 3 + k) +=
// *(field + j * 3 + k) * *(charges + j) / *(masses + j) * scaling_factor;
// *(particles + j * 3 + k) +=
// *(velocities + j * 3 + k) * time_step;
// if (*(periodicity + k) == 1)
// {
// if (*(particles + j * 3 + k) > INTERFACE_TEST_BOX_SIZE)
// *(particles + j * 3 + k) -= INTERFACE_TEST_BOX_SIZE;
// else if (*(particles + j * 3 + k) < 0.0l)
// *(particles + j * 3 + k) += INTERFACE_TEST_BOX_SIZE;
// }
// }
// }
if ((i + 1) % INTERFACE_TEST_XYZ_INTERVAL == 0)
{
for (j = 0; j < comm_size; ++j)
{
if (j == comm_rank)
{
if ((j == 0) && ((i + 1) == INTERFACE_TEST_XYZ_INTERVAL))
xyz_file = fopen ("C_test.xyz", "w");
else
xyz_file = fopen ("C_test.xyz", "a");
if (j == 0)
{
fprintf (xyz_file, "%" FCS_LMOD_INT "d\n%" FCS_LMOD_INT "d %s %s %d %" FCS_LMOD_INT "d %c %c %c\n",
number_of_particles, i+1, "C", (i%4==0)?"pp3mg":(i%4==1)?"fmm":(i%4==2)?"p2nfft":(i%4==3)?"p3m":"unknown", comm_size,
number_of_runs,
(*periodicity == 1) ? 'T' : 'F',
(*(periodicity + 1) == 1) ? 'T' : 'F',
(*(periodicity + 2) == 1) ? 'T' : 'F');
}
for (k = 0; k < local_particles; ++k)
{
fprintf (xyz_file, "O %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e\n",
*(particles + 3 * k), *(particles + 3 * k + 1),
*(particles + 3 * k + 2),
*(charges + k),
*(velocities + 3 * k),
*(velocities + 3 * k + 1),
*(velocities + 3 * k + 2));
/**(potentials + k),
*(field + 3 * k),
*(field + 3 * k + 1),
*(field + 3 * k + 2));*/
}
fclose (xyz_file);
}
MPI_Barrier (communicator);
}
}
}
// if (comm_rank == 0)
// printf ("\n");
// if (comm_rank == 0)
// printf ("Parameters... \n");
// if (comm_rank == 0)
// fcs_print_parameters (handle);
if (comm_rank == 0)
fcs_result_print_result (result);
fcs_result_destroy (result);
free (box_a);
free (box_b);
free (box_c);
free (offset);
#if FCS_ENABLE_PP3MG
fcs_destroy (handle_pp3mg);
#endif
#if FCS_ENABLE_FMM
fcs_destroy (handle_fmm);
#endif
#if FCS_ENABLE_P2NFFT
fcs_destroy (handle_p2nfft);
#endif
#if FCS_ENABLE_P3M
fcs_destroy (handle_p3m);
#endif
/* printf("address of handle: %p\n",*fcs_get_box_a(handle)); */
free (particles);
free (charges);
free (velocities);
free (field);
free (potentials);
free (masses);
MPI_Finalize ();
return (FCS_SUCCESS);
}
int main(int argc, char **argv)
{
const char* method = "mmm1d";
const char* datafile = "../inp_data/mmm/mmm1d_wall.dat";
MPI_Comm comm = MPI_COMM_WORLD;
fcs_int comm_rank, comm_size;
MPI_Init(&argc, &argv);
MPI_Comm_size(comm, &comm_size);
MPI_Comm_rank(comm, &comm_rank);
fcs_int periodicity[3] = { 0, 0, 1 };
fcs_int node_grid[3] = {1, 1, comm_size};
fcs_int node[3];
if (comm_rank == 0)
fprintf(stderr, "Creating cartesian communicator...\n");
MPI_Dims_create(comm_size, 3, node_grid);
MPI_Cart_create(MPI_COMM_WORLD, 3, node_grid, periodicity, 1, &comm);
if (comm_rank == 0)
fprintf(stderr, " node_grid=(%d, %d, %d)\n", node_grid[0], node_grid[1], node_grid[2]);
MPI_Cart_coords(comm, comm_rank, 3, node);
fcs_int pid;
if (comm_rank == 0) {
printf("------------------------------\n");
printf("Running mmm1d test on %d nodes\n", comm_size);
printf("------------------------------\n");
}
fcs_float box_l[3] = { 10.0, 10.0, 10.0 };
fcs_float offset[3] = {0.0, 0.0, 0.0};
fcs_int n_particles=0, total_particles = 4;
fcs_float charges[total_particles];
fcs_float positions[3*total_particles];
fcs_float forces[3*total_particles];
fcs_float potentials[total_particles];
fcs_float charge_sum = 0.0;
if (comm_rank == 0) {
printf("Reading %s...\n", datafile);
FILE *data = fopen(datafile, "r");
if (!data) {
fprintf(stderr, "ERROR: Can't read %s!\n", datafile);
perror("ERROR");
exit(1);
}
for (pid = 0; pid < total_particles; pid++) {
fscanf(data, "%lf %lf %lf", &positions[3*pid], &positions[3*pid+1], &positions[3*pid+2]);
fscanf(data, "%lf", &charges[pid]);
printf("read: %d %le %le %le %le\n", pid, charges[pid], positions[3*pid], positions[3*pid+1], positions[3*pid+2]);
charge_sum += charges[pid];
}
fclose(data);
n_particles=total_particles;
/*
printf("Charge 1 with value %e, position %e %e %e\n",charges[0],positions[0],positions[1],positions[2]);
printf("Charge %d with value %e, position %e %e %e\n", n_particles,charges[n_particles-1],positions[3*n_particles-3],positions[3*n_particles-2],positions[3*n_particles-1]);
*/
} else n_particles=0;
FCS handle = NULL;
FCSResult result = NULL;
if (comm_rank == 0) printf("\nInitializing mmm1d...\n");
result = fcs_init(&handle, method, comm);
assert_fcs(result);
if (comm_rank == 0) printf("\nSetting parameters...\n");
fcs_float box_a[3] = { 0.0, 0.0, 0.0 };
fcs_float box_b[3] = { 0.0, 0.0, 0.0 };
fcs_float box_c[3] = { 0.0, 0.0, 0.0 };
box_a[0] = box_l[0];
box_b[1] = box_l[1];
box_c[2] = box_l[2];
result = fcs_set_common(handle, 1, box_a, box_b, box_c,
offset, periodicity, total_particles);
if (result != NULL) {
fcs_result_print_result(result);
MPI_Finalize();
exit(1);
}
assert_fcs(result);
/* Tuning */
if (comm_rank == 0) printf("\nTesting parameters interface...\n");
fcs_float val;
fcs_int vali;
fcs_mmm1d_get_far_switch_radius(handle, &val);
if (comm_rank == 0) printf("default far switch radius: %f\n",val);
fcs_mmm1d_set_far_switch_radius(handle, 6.0);
fcs_mmm1d_get_far_switch_radius(handle, &val);
if (comm_rank == 0) printf("new far switch radius: %f\n",val);
fcs_mmm1d_get_bessel_cutoff(handle, &vali);
if (comm_rank == 0) printf("default bessel_cutoff: %d\n",vali);
fcs_mmm1d_set_bessel_cutoff(handle, 3);
fcs_mmm1d_get_bessel_cutoff(handle, &vali);
if (comm_rank == 0) printf("new bessel_cutoff: %d\n",vali);
fcs_mmm1d_get_maxPWerror(handle, &val);
if (comm_rank == 0) printf("default maxPWerror: %f\n",val);
fcs_mmm1d_set_maxPWerror(handle, 0.0001);
fcs_mmm1d_get_maxPWerror(handle, &val);
if (comm_rank == 0) printf("new maxPWerror: %f\n",val);
if (comm_rank == 0) printf("\nTesting tuning routine...\n");
result = fcs_tune(handle, n_particles, positions, charges);
if (result != NULL) {
fcs_result_print_result(result);
MPI_Finalize();
exit(1);
}
//fcs_mmm1d_get_bessel_cutoff(handle, &vali);
//printf("calculated bessel cutoff: %d\n",vali);
if (comm_rank == 0) printf("\nTesting running routine...\n");
result = fcs_run(handle, n_particles, positions, charges, forces, potentials);
assert_fcs(result);
MPI_Barrier(comm);
if (comm_rank == 0) {
for(pid=0; pid<total_particles; pid++) {
printf("Potential on particle %d: %e\n", pid, potentials[pid]);
//printf("Potential on particle 2: %e\n", potentials[1]);
printf("Force on particle %d: %e, %e, %e\n", pid, forces[3*pid], forces[3*pid+1], forces[3*pid+2]);
//printf("Force on particle 2: %e, %e, %e\n", forces[3], forces[4], forces[5]);
}
printf("\nFinalizing...\n");
}
fcs_destroy(handle);
MPI_Finalize();
if (comm_rank == 0) printf("Done.\n");
return 0;
}
/**
* Fortran wrapper function ot initialize an FCS solver method
*/
FCSResult fcs_init_f(FCS *handle, const char *method_name, MPI_Fint communicator)
{
MPI_Comm c_comm = MPI_Comm_f2c(communicator);
return fcs_init(handle, method_name, c_comm);
}
int main(int argc, char **argv)
{
int comm_rank, comm_size;
const char* method = "p2nfft";
/* DEBUG */
// const char* datafile = "../inp_data/p2nfft/debug_wall_small.dat";
const char* datafile = "../inp_data/p3m/p3m_wall.dat";
MPI_Comm comm = MPI_COMM_WORLD;
fcs_int periodicity[3] = { 1, 1, 1 };
fcs_int pid;
fcs_float tolerance = 1.e-3;
MPI_Init(&argc, &argv);
MPI_Comm_size(comm, &comm_size);
MPI_Comm_rank(comm, &comm_rank);
if(!comm_rank){
printf("-------------------\n");
printf("Running p2nfft test\n");
printf("-------------------\n");
}
fcs_float box_l[3] = { 10.0, 10.0, 10.0 };
fcs_float offset[3] = {0.0, 0.0, 0.0};
/* DEBUG */
// int n_particles = 4;
int n_particles = 300;
fcs_float charges[300];
fcs_float positions[900];
fcs_float far_fields[900];
fcs_float forces[900];
fcs_float reference_forces[900];
fcs_float far_potentials[300];
fcs_float energies[300];
fcs_float virial[9];
fcs_int local_particles = 0;
fcs_float local_charges[300];
fcs_float local_positions[900];
fcs_int global_particle_indices[300];
if(!comm_rank)
printf("Reading %s...\n", datafile);
FILE *data = fopen(datafile, "r");
if (!data) {
fprintf(stderr, "ERROR: Can't read %s!", datafile);
perror("ERROR");
exit(1);
}
fcs_float charge_sum = 0.0;
for (pid = 0; pid < n_particles; pid++) {
fscanf(data, "%" FCS_CONV_FLOAT "f %" FCS_CONV_FLOAT "f %" FCS_CONV_FLOAT "f",
&positions[3*pid], &positions[3*pid+1], &positions[3*pid+2]);
fscanf(data, "%" FCS_CONV_FLOAT "f",
&charges[pid]);
fscanf(data, "%" FCS_CONV_FLOAT "f %" FCS_CONV_FLOAT "f %" FCS_CONV_FLOAT "f",
&reference_forces[3*pid], &reference_forces[3*pid+1], &reference_forces[3*pid+2]);
charge_sum += charges[pid];
}
fclose(data);
FCS handle = NULL;
FCSResult result = NULL;
MPI_Barrier(MPI_COMM_WORLD);
if(!comm_rank)
printf("Initializing p2nfft...\n");
result = fcs_init(&handle, method, comm);
assert_fcs(result);
if(!comm_rank)
printf("Reading particles ... \n");
for (pid = 0; pid < n_particles; ++pid) {
if (pid % comm_size == comm_rank) {
local_charges[local_particles] = charges[pid];
local_positions[3*local_particles] = positions[3*pid];
local_positions[3*local_particles+1] = positions[3*pid+1];
local_positions[3*local_particles+2] = positions[3*pid+2];
global_particle_indices[local_particles] = pid;
++local_particles;
}
}
if(!comm_rank)
printf("Setting parameters...\n");
fcs_float box_a[3] = { 0.0, 0.0, 0.0 };
fcs_float box_b[3] = { 0.0, 0.0, 0.0 };
fcs_float box_c[3] = { 0.0, 0.0, 0.0 };
box_a[0] = box_l[0];
box_b[1] = box_l[1];
box_c[2] = box_l[2];
result = fcs_set_common(handle, 1, box_a, box_b, box_c,
offset, periodicity, n_particles);
assert_fcs(result);
/* Tuning */
fcs_set_tolerance(handle, FCS_TOLERANCE_TYPE_FIELD, tolerance);
if(!comm_rank)
printf("Tuning p2nfft to tolerance %" FCS_LMOD_FLOAT "e...\n", tolerance);
result = fcs_tune(handle, local_particles, local_positions, local_charges);
assert_fcs(result);
/* activate virial computation */
result = fcs_set_compute_virial(handle, 1);
assert_fcs(result);
/* Far field computation */
if(!comm_rank)
printf("Running p2nfft (computing far fields and potentials)...\n");
result = fcs_run(handle, local_particles,
local_positions, local_charges, far_fields, far_potentials);
assert_fcs(result);
/* get and print virial */
result = fcs_get_virial(handle, virial);
assert_fcs(result);
if(!comm_rank)
printf("virial tensor = [%" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e; %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e; %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e]\n", virial[0], virial[1], virial[2],
virial[3], virial[4], virial[5], virial[6], virial[7], virial[8]);
/* Add components */
for (pid = 0; pid < local_particles; pid++) {
forces[3*pid] = local_charges[pid] * far_fields[3*pid];
forces[3*pid+1] = local_charges[pid] * far_fields[3*pid+1];
forces[3*pid+2] = local_charges[pid] * far_fields[3*pid+2];
energies[pid] = 0.5 * local_charges[pid] * far_potentials[pid];
}
/* Compare forces to reference field */
fcs_float sqr_sum = 0.0;
fcs_float sum_energy = 0.0;
for (pid = 0; pid < local_particles; pid++) {
sum_energy += energies[pid];
fcs_float d0 = forces[3*pid] - reference_forces[3*global_particle_indices[pid]];
fcs_float d1 = forces[3*pid+1] - reference_forces[3*global_particle_indices[pid]+1];
fcs_float d2 = forces[3*pid+2] - reference_forces[3*global_particle_indices[pid]+2];
sqr_sum += d0*d0+d1*d1+d2*d2;
}
/* Reduce to global values */
fcs_float global_total_energy, global_sqr_sum;
MPI_Reduce(&sum_energy, &global_total_energy, 1, FCS_MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&sqr_sum, &global_sqr_sum, 1, FCS_MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
if (!comm_rank) {
printf("sum_energy=%" FCS_LMOD_FLOAT "f\n", global_total_energy);
printf("rms_error=%e\n", sqrt(global_sqr_sum / (fcs_float)n_particles));
}
if(!comm_rank)
printf("Finalizing...\n");
fcs_destroy(handle);
MPI_Finalize();
if(!comm_rank)
printf("Done.\n");
return 0;
}
int main(int argc, char* argv[])
{
MPI_Comm comm;
int comm_size, comm_rank;
fcs_float *x, *q, *f, *p;
fcs_int n_axis, n_total, n_local, n_local_max;
fcs_int i, j, k;
fcs_int p_c, p_start, p_stop,ip;
fcs_float e_local, e_total;
fcs_float madelung_approx;
const fcs_float madelung = 1.74756459463318219;
int mpi_thread_requested = MPI_THREAD_MULTIPLE;
int mpi_thread_provided;
FCS fcs_handle;
FCSResult fcs_result;
char method[] = "pepc";
fcs_float box_a[] = { 1.0, 0.0, 0.0 };
fcs_float box_b[] = { 0.0, 1.0, 0.0 };
fcs_float box_c[] = { 0.0, 0.0, 1.0 };
fcs_float offset[] = { 0.0, 0.0, 0.0 };
fcs_int periodic[] = { 1, 1, 1 };
//fcs_int periodic[] = { 0, 0, 0 };
fcs_float theta = 0.2;
fcs_float epsilon = 1.23e-6;
MPI_Init_thread(&argc, &argv, mpi_thread_requested, &mpi_thread_provided);
comm = MPI_COMM_WORLD;
MPI_Comm_size(comm, &comm_size);
MPI_Comm_rank(comm, &comm_rank);
if (mpi_thread_provided < mpi_thread_requested && comm_rank == 0) {
printf("Call to MPI_INIT_THREAD failed. Requested/provided level of multithreading: %d / %d. Continuing but expect program crash.\n", mpi_thread_requested, mpi_thread_provided);
}
n_axis = 16;
n_total = n_axis * n_axis * n_axis;
n_local = n_total / comm_size;
if(comm_rank == comm_size-1) n_local += n_total % comm_size;
n_local_max = n_total / comm_size + n_total % comm_size;
if (comm_rank == 0) {
printf("*** RUNNING pepc TEST ***\n");
printf(" n_total = %" FCS_LMOD_INT "d\n", n_total);
printf(" n_procs = %d\n", comm_size);
printf(" theta = %e\n", theta);
printf(" epsilon = %e\n", epsilon);
}
x = (fcs_float*)malloc(3 * n_local * sizeof(fcs_float));
q = (fcs_float*)malloc( n_local * sizeof(fcs_float));
f = (fcs_float*)malloc(3 * n_local * sizeof(fcs_float));
p = (fcs_float*)malloc( n_local * sizeof(fcs_float));
p_c = 0;
p_start = comm_rank*(n_total/comm_size);
p_stop = p_start + n_local;
for (ip=p_start; ip<p_stop; ip++, p_c++) {
i = ip % n_axis;
j = (ip / n_axis) % n_axis;
k = ip / (n_axis*n_axis);
x[3*p_c ] = offset[0] + (i * box_a[0]) / n_axis;
x[3*p_c+1] = offset[1] + (j * box_b[1]) / n_axis;
x[3*p_c+2] = offset[2] + (k * box_c[2]) / n_axis;
q[p_c] = ((i+j+k)%2 ? 1.0 : -1.0);
/* printf("init positions (rank %d) for particle id %d: %e %e %e %e\n", */
/* comm_rank, ip, x[3*p_c ], x[3*p_c+1], x[3*p_c+2], q[p_c]); */
}
fcs_result = fcs_init(&fcs_handle, method, comm);
assert_fcs(fcs_result);
fcs_result = fcs_set_common(fcs_handle, 1,
box_a, box_b, box_c,
offset, periodic, n_total);
assert_fcs(fcs_result);
fcs_result = fcs_pepc_setup(fcs_handle, epsilon, theta);
assert_fcs(fcs_result);
fcs_result = fcs_tune(fcs_handle, n_local, n_local_max, x, q);
assert_fcs(fcs_result);
fcs_result = fcs_run(fcs_handle, n_local, n_local_max, x, q, f, p);
assert_fcs(fcs_result);
e_local = 0.0;
for (i=0; i<n_local; ++i)
e_local += p[i] * q[i];
MPI_Reduce(&e_local, &e_total, 1, MPI_DOUBLE, MPI_SUM, 0, comm);
//madelung_approx = 8.0 * M_PI / n_axis * e_total / n_total;
madelung_approx = 1.0 / n_axis * e_total / n_total;
if (comm_rank == 0) {
printf("\n");
printf(" Results:\n");
printf(" Energy: %e\n", e_total);
printf(" Madelung constant: %e\n", madelung_approx);
printf(" Relative error: %e\n", fabs(madelung-fabs(madelung_approx))/madelung);
}
p_c = 0;
p_start = comm_rank*(n_total/comm_size);
p_stop = p_start + n_local;
for (ip=p_start; ip<p_stop; ip++, p_c++) {
/* printf("results (rank %d) for particle id %d: %e %e %e %e\n", */
/* comm_rank, ip, f[3*p_c ], f[3*p_c+1], f[3*p_c+2], p[p_c]); */
/* printf("dataout: %e %e %e %e %e %e %e %e\n", x[3*p_c ], x[3*p_c+1], x[3*p_c+2], q[p_c], */
/* f[3*p_c ], f[3*p_c+1], f[3*p_c+2], p[p_c]); */
}
fcs_destroy(fcs_handle);
free(x);
free(q);
free(f);
free(p);
if (comm_rank == 0)
printf("*** pepc DONE ***\n");
MPI_Finalize();
return 0;
}
void init_fcs(void) {
FCSResult res;
fcs_int srf = 1;
char *method;
fcs_int pbc [3] = { pbc_dirs.x, pbc_dirs.y, pbc_dirs.z };
fcs_float BoxX[3] = { box_x.x, box_x.y, box_x.z };
fcs_float BoxY[3] = { box_y.x, box_y.y, box_y.z };
fcs_float BoxZ[3] = { box_z.x, box_z.y, box_z.z };
fcs_float off [3] = { 0.0, 0.0, 0.0 };
/* subtract CM momentum */
if (0 == imdrestart) {
int i, k; real ptot[4], ptot_2[4], px, py, pz;
ptot[0] = 0.0; ptot[1] = 0.0; ptot[2] = 0.0, ptot[3] = 0.0;
for (k=0; k<NCELLS; ++k) { /* loop over all cells */
cell *p = CELLPTR(k);
for (i=0; i<p->n; i++) {
ptot[0] += IMPULS(p,i,X);
ptot[1] += IMPULS(p,i,Y);
ptot[2] += IMPULS(p,i,Z);
ptot[3] += MASSE(p,i);
}
}
#ifdef MPI
MPI_Allreduce( ptot, ptot_2, 4, REAL, MPI_SUM, cpugrid);
ptot[0] = ptot_2[0];
ptot[1] = ptot_2[1];
ptot[2] = ptot_2[2];
ptot[3] = ptot_2[3];
#endif
px = ptot[0]/ptot[3];
py = ptot[1]/ptot[3];
pz = ptot[2]/ptot[3];
for (k=0; k<NCELLS; ++k) { /* loop over all cells */
cell *p = CELLPTR(k);
for (i=0; i<p->n; i++) {
IMPULS(p,i,X) -= px * MASSE(p,i);
IMPULS(p,i,Y) -= py * MASSE(p,i);
IMPULS(p,i,Z) -= pz * MASSE(p,i);
}
}
}
switch (fcs_method) {
case FCS_METH_DIRECT: method = "direct"; break;
case FCS_METH_PEPC: method = "pepc"; break;
case FCS_METH_FMM: method = "fmm"; break;
case FCS_METH_P3M: method = "p3m"; srf = fcs_near_field_flag; break;
case FCS_METH_P2NFFT: method = "p2nfft"; srf = fcs_near_field_flag; break;
case FCS_METH_VMG: method = "vmg"; break;
case FCS_METH_PP3MG: method = "pp3mg"; break;
}
/* initialize handle and set common parameters */
res = fcs_init(&handle, method, cpugrid);
ASSERT_FCS(res);
res = fcs_set_common(handle, srf, BoxX, BoxY, BoxZ, off, pbc, natoms);
ASSERT_FCS(res);
res = fcs_require_virial(handle, 1);
ASSERT_FCS(res);
/* set method specific parameters */
switch (fcs_method) {
#ifdef FCS_ENABLE_DIRECT
case FCS_METH_DIRECT:
/* nothing to do */
break;
#endif
#ifdef FCS_ENABLE_PEPC
case FCS_METH_PEPC:
res = fcs_pepc_setup(handle, (fcs_float)fcs_pepc_eps,
(fcs_float)fcs_pepc_theta );
ASSERT_FCS(res);
res = fcs_pepc_set_num_walk_threads( handle, (fcs_int)fcs_pepc_nthreads );
ASSERT_FCS(res);
break;
#endif
#ifdef FCS_ENABLE_FMM
case FCS_METH_FMM:
res = fcs_fmm_set_absrel(handle, (fcs_int)fcs_fmm_absrel);
ASSERT_FCS(res);
res = fcs_fmm_set_tolerance_energy(handle, (fcs_float)fcs_tolerance);
ASSERT_FCS(res);
break;
#endif
#ifdef FCS_ENABLE_P3M
case FCS_METH_P3M:
if (0==srf) {
res = fcs_p3m_set_r_cut(handle, (fcs_float)fcs_rcut);
ASSERT_FCS(res);
}
res = fcs_set_tolerance(handle, FCS_TOLERANCE_TYPE_FIELD,
(fcs_float)fcs_tolerance);
ASSERT_FCS(res);
if (fcs_grid_dim.x)
res = fcs_p3m_set_grid(handle, (fcs_int)fcs_grid_dim.x);
ASSERT_FCS(res);
break;
#endif
#ifdef FCS_ENABLE_P2NFFT
case FCS_METH_P2NFFT:
if (0==srf) {
res = fcs_p2nfft_set_r_cut(handle, (fcs_float)fcs_rcut);
ASSERT_FCS(res);
}
res = fcs_set_tolerance(handle, FCS_TOLERANCE_TYPE_FIELD,
(fcs_float)fcs_tolerance);
ASSERT_FCS(res);
if (fcs_grid_dim.x) {
res = fcs_p2nfft_set_grid(handle, (fcs_int)fcs_grid_dim.x,
(fcs_int)fcs_grid_dim.y, (fcs_int)fcs_grid_dim.z);
ASSERT_FCS(res);
}
if (fcs_p2nfft_intpol_order) {
res = fcs_p2nfft_set_pnfft_interpolation_order(handle,
(fcs_int)fcs_p2nfft_intpol_order);
ASSERT_FCS(res);
}
if (fcs_p2nfft_epsI) {
res = fcs_p2nfft_set_epsI(handle, (fcs_float)fcs_p2nfft_epsI);
ASSERT_FCS(res);
}
//res = fcs_p2nfft_set_pnfft_window_by_name(handle, "bspline");
//ASSERT_FCS(res);
break;
#endif
#ifdef FCS_ENABLE_VMG
case FCS_METH_VMG:
if (fcs_vmg_near_field_cells) {
res = fcs_vmg_set_near_field_cells(handle, (fcs_int)fcs_vmg_near_field_cells);
ASSERT_FCS(res);
}
if (fcs_vmg_interpol_order) {
res = fcs_vmg_set_interpolation_order(handle, (fcs_int)fcs_vmg_interpol_order);
ASSERT_FCS(res);
}
if (fcs_vmg_discr_order) {
res = fcs_vmg_set_discretization_order(handle, (fcs_int)fcs_vmg_discr_order);
ASSERT_FCS(res);
}
if (fcs_iter_tolerance > 0) {
res = fcs_vmg_set_precision(handle, (fcs_float)fcs_iter_tolerance);
ASSERT_FCS(res);
}
break;
#endif
#ifdef FCS_ENABLE_PP3MG
case FCS_METH_PP3MG:
/* use default values, if not specified otherwise */
if (fcs_grid_dim.x) {
res = fcs_pp3mg_set_cells_x(handle, (fcs_int)fcs_grid_dim.x);
ASSERT_FCS(res);
res = fcs_pp3mg_set_cells_y(handle, (fcs_int)fcs_grid_dim.y);
ASSERT_FCS(res);
res = fcs_pp3mg_set_cells_z(handle, (fcs_int)fcs_grid_dim.z);
ASSERT_FCS(res);
}
if (fcs_pp3mg_ghosts) {
res = fcs_pp3mg_set_ghosts(handle, (fcs_int)fcs_pp3mg_ghosts);
ASSERT_FCS(res);
}
if (fcs_pp3mg_degree) {
res = fcs_pp3mg_set_degree(handle, (fcs_int)fcs_pp3mg_degree);
ASSERT_FCS(res);
}
if (fcs_pp3mg_max_part) {
res = fcs_pp3mg_set_max_particles(handle, (fcs_int)fcs_pp3mg_max_part);
ASSERT_FCS(res);
}
if (fcs_max_iter) {
res = fcs_pp3mg_set_max_iterations(handle,(fcs_int)fcs_max_iter);
ASSERT_FCS(res);
}
if (fcs_iter_tolerance > 0) {
res = fcs_pp3mg_set_tol(handle, (fcs_float)fcs_iter_tolerance);
ASSERT_FCS(res);
}
break;
#endif
default:
error("FCS method unknown or not implemented");
break;
}
pack_fcs();
res = fcs_tune(handle, nloc, nloc_max, pos, chg);
ASSERT_FCS(res);
/* inform about tuned parameters */
switch (fcs_method) {
fcs_int grid_dim[3];
fcs_float r_cut;
#ifdef FCS_ENABLE_P3M
case FCS_METH_P3M:
res = fcs_p3m_get_r_cut(handle, &r_cut);
ASSERT_FCS(res);
res = fcs_p3m_get_grid(handle, grid_dim);
ASSERT_FCS(res);
if (0==myid)
printf("FCS: Tuned grid dimensions, cutoff: %d %d %d, %f\n",
grid_dim[0], grid_dim[1], grid_dim[2], r_cut);
break;
#endif
#ifdef FCS_ENABLE_P2NFFT
case FCS_METH_P2NFFT:
res = fcs_p2nfft_get_grid(handle, grid_dim, grid_dim+1, grid_dim+2);
ASSERT_FCS(res);
res = fcs_p2nfft_get_r_cut(handle, &r_cut);
ASSERT_FCS(res);
if (0==myid)
printf("FCS: Tuned grid dimensions, cutoff: %d %d %d, %f\n",
grid_dim[0], grid_dim[1], grid_dim[2], r_cut);
break;
#endif
#ifdef FCS_ENABLE_PP3MG
case FCS_METH_PP3MG:
res = fcs_pp3mg_get_cells_x(handle, grid_dim );
ASSERT_FCS(res);
res = fcs_pp3mg_get_cells_y(handle, grid_dim+1);
ASSERT_FCS(res);
res = fcs_pp3mg_get_cells_z(handle, grid_dim+2);
if (0==myid)
printf("FCS: Tuned grid dimensions: %d %d %d\n",
grid_dim[0], grid_dim[1], grid_dim[2]);
ASSERT_FCS(res);
break;
#endif
default:
break;
}
/* add near-field potential, after fcs_tune */
if (0==srf) fcs_update_pottab();
}
int main(int argc, char* argv[])
{
MPI_Comm comm;
int comm_size, comm_rank;
fcs_float *x, *q, *f, *p;
fcs_int n_axis, n_total, n_local, n_local_max;
fcs_int i, j, k;
fcs_int p_c;
FCS fcs_handle;
FCSResult fcs_result;
char method[] = "vmg";
fcs_float box_a[] = { 1.0, 0.0, 0.0 };
fcs_float box_b[] = { 0.0, 1.0, 0.0 };
fcs_float box_c[] = { 0.0, 0.0, 1.0 };
fcs_float offset[] = { 0.0, 0.0, 0.0 };
fcs_int periodic[] = { 0, 0, 0 };
fcs_int max_level = 6;
fcs_int max_iterations = 20;
fcs_int smoothing_steps = 3;
fcs_int cycle_type = 1;
fcs_float precision = 1e-10;
fcs_int near_field_cells = 6;
fcs_int interpolation_degree = 4;
fcs_int discretization_order = 2;
MPI_Init(&argc, &argv);
comm = MPI_COMM_WORLD;
MPI_Comm_size(comm, &comm_size);
MPI_Comm_rank(comm, &comm_rank);
n_axis = 16;
n_total = n_axis * n_axis * n_axis;
if (comm_rank == 0) {
n_local = n_total;
n_local_max = n_total;
}else {
n_local = 0;
n_local_max = 0;
}
if (comm_rank == 0) {
printf("*** RUNNING vmg TEST ***\n");
printf(" n_total = %" FCS_LMOD_INT "d\n", n_total);
printf(" n_procs = %d\n", comm_size);
printf(" periodicity_x = %d\n", periodic[0]);
printf(" periodicity_y = %d\n", periodic[1]);
printf(" periodicity_z = %d\n", periodic[2]);
printf(" max_level = %d\n", max_level);
printf(" max_iterations = %d\n", max_iterations);
printf(" smoothing_steps = %d\n", smoothing_steps);
printf(" cycle_type = %d\n", cycle_type);
printf(" precision = %e\n", precision);
printf(" near_field_cells = %d\n", near_field_cells);
printf(" interpolation_degree = %d\n", interpolation_degree);
printf(" discretization_order = %d\n", discretization_order);
}
x = (fcs_float*)malloc(3 * n_local_max * sizeof(fcs_float));
q = (fcs_float*)malloc(n_local_max * sizeof(fcs_float));
f = (fcs_float*)malloc(3 * n_local_max * sizeof(fcs_float));
p = (fcs_float*)malloc(n_local_max * sizeof(fcs_float));
if (comm_rank == 0) {
p_c = 0;
for (i=0; i<n_axis; ++i)
for (j=0; j<n_axis; ++j)
for (k=0; k<n_axis; ++k) {
x[3*p_c ] = offset[0] + (i * box_a[0]) / n_axis;
x[3*p_c+1] = offset[1] + (j * box_b[1]) / n_axis;
x[3*p_c+2] = offset[2] + (k * box_c[2]) / n_axis;
q[p_c] = ((i+j+k)%2 ? 1.0 : -1.0);
++p_c;
}
}
fcs_result = fcs_init(&fcs_handle, method, comm);
assert_fcs(fcs_result);
fcs_result = fcs_set_common(fcs_handle, 1,
box_a, box_b, box_c,
offset, periodic, n_total);
assert_fcs(fcs_result);
fcs_result = fcs_vmg_setup(fcs_handle, max_level,
max_iterations, smoothing_steps,
cycle_type, precision, near_field_cells,
interpolation_degree, discretization_order);
assert_fcs(fcs_result);
fcs_result = fcs_tune(fcs_handle, n_local, n_local_max, x, q);
assert_fcs(fcs_result);
fcs_result = fcs_run(fcs_handle, n_local, n_local_max, x, q, f, p);
assert_fcs(fcs_result);
fcs_destroy(fcs_handle);
free(x);
free(q);
free(f);
free(p);
if (comm_rank == 0)
printf("*** vmg DONE ***\n");
MPI_Finalize();
return 0;
}
int main (int argc, char **argv)
{
char input_file_name[300], *conf_file_name;
FILE *input_file, *conf_file;
int i, j, help, p_local = 0;
FCSResult fcs_result;
FCS fcs_handle;
char parameterstring[200];
int particles_i;
fcs_int particles = 0;
fcs_int local_particles = 0;
fcs_float *pos = NULL;
fcs_float *charges = NULL;
fcs_float *field = NULL;
fcs_float *potentials = NULL;
int cells_x = 128, cells_y = 128, cells_z = 128;
int periodic = 1;
int ghosts = 4;
int degree = 4;
int max_particles_i;
fcs_int max_particles = 2000000;
int max_iterations;
double err_bound = 1.e-3;
int nu1, nu2;
fcs_float box[DIM][DIM] = {
{1.0, 0.0, 0.0},
{0.0, 1.0, 0.0},
{0.0, 0.0, 1.0}};
fcs_float offset[DIM] = {0.0, 0.0, 0.0};
fcs_int periodic_flags[DIM] = {1,1,1};
/* Variables for analyzing results */
int read_ret_value = 0;
double f_sum_local[DIM];
double f_sum[DIM];
double f_sum_squared_local[DIM];
double f_sum_squared[DIM];
double f_max_abs_local[DIM];
double f_max_abs[DIM];
double e_sum_local = 0.0;
double e_sum = 0.0;
int min = 0;
/* Variables for reading data */
double my_x, my_y, my_z, my_q;
/* Size of local domain */
double x_start, y_start, z_start;
double x_end, y_end, z_end;
int my_rank;
int mpi_size;
MPI_Comm mpi_comm_cart;
int mpi_dims_i[DIM];
fcs_int mpi_dims[DIM];
int mpi_periods[DIM];
int mpi_coords[DIM];
MPI_Status status;
double starttime = 0.0, endtime = 0.0;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
if (my_rank == 0) {
fprintf(stderr, "----------------\n");
fprintf(stderr, "Running pp3mg test\n");
fprintf(stderr, "----------------\n");
fprintf(stderr, "Setting up MPI...\n");
fprintf(stderr, " Using %d tasks.\n", mpi_size);
}
if(argc == 1) {
printf("No config file was specified!\n");
MPI_Finalize();
exit(-1);
}
/* read config and input files */
if(argc >= 2) {
conf_file_name = argv[1];
read_ret_value = read_config_file(conf_file_name,
input_file_name,
&cells_x, &cells_y, &cells_z,
&max_iterations, &max_particles_i,
&nu1, &nu2, &ghosts, &err_bound);
if(read_ret_value) {
printf("Config file couldn't be read!\n");
MPI_Finalize();
exit(-1);
}
max_particles = max_particles_i;
}
if((input_file = fopen(input_file_name, "r")) == NULL) {
printf("input file name: %s\n" , input_file_name);
printf("No input file was found!\n");
MPI_Finalize();
exit(-1);
}
fscanf(input_file, " %d", &particles_i);
particles = particles_i;
/* create a cartesian communicator */
for(i = 0; i < DIM; ++i) {
mpi_periods[i] = 1;
mpi_dims_i[i] = 0;
}
assert(MPI_Dims_create(mpi_size, 3, mpi_dims_i) == MPI_SUCCESS);
assert(MPI_Cart_create(MPI_COMM_WORLD, 3, mpi_dims_i, mpi_periods, 1, &mpi_comm_cart) == MPI_SUCCESS);
assert(MPI_Comm_rank(mpi_comm_cart, &my_rank) == MPI_SUCCESS);
assert(MPI_Cart_coords(mpi_comm_cart, my_rank, 3, mpi_coords) == MPI_SUCCESS);
for (i = 0; i < DIM; ++i) mpi_dims[i] = mpi_dims_i[i];
if (mpi_comm_cart != MPI_COMM_NULL) {
/* initialize parameters for particle grid */
x_start = ((double)1.)/mpi_dims[0] * (double)mpi_coords[0];
x_end = ((double)1.)/mpi_dims[0] * (double)(mpi_coords[0]+1.);
y_start = ((double)1.)/mpi_dims[1] * (double)mpi_coords[1];
y_end = ((double)1.)/mpi_dims[1] * (double)(mpi_coords[1]+1.);
z_start = ((double)1.)/mpi_dims[2] * (double)mpi_coords[2];
z_end = ((double)1.)/mpi_dims[2] * (double)(mpi_coords[2]+1.);
local_particles = 0;
/* count local partciles , domain decomposition */
for(i = 0; i < particles; ++i) {
fscanf(input_file, " %lf", &my_x);
fscanf(input_file, " %lf", &my_y);
fscanf(input_file, " %lf", &my_z);
fscanf(input_file, " %lf", &my_q);
if(x_start <= my_x && my_x < x_end &&
y_start <= my_y && my_y < y_end &&
z_start <= my_z && my_z < z_end)
local_particles ++;
}
}
fclose(input_file);
if(local_particles != 0) {
pos = malloc(local_particles*DIM*sizeof(fcs_float));
assert(pos);
charges = malloc(local_particles*sizeof(fcs_float));
assert(charges);
field = malloc(local_particles*DIM*sizeof(fcs_float));
assert(field);
potentials = malloc(local_particles*sizeof(fcs_float));
assert(potentials);
/* open the input file again and read particle data */
if((input_file = fopen(input_file_name, "r")) == NULL) {
printf("No input file found!\n");
MPI_Finalize();
exit(-1);
}
fscanf(input_file, " %d", &particles);
p_local = 0;
for(i = 0; i < particles; ++i ) {
fscanf(input_file, " %lf", &my_x);
fscanf(input_file, " %lf", &my_y);
fscanf(input_file, " %lf", &my_z);
fscanf(input_file, " %lf", &my_q);
if(x_start <= my_x && my_x < x_end &&
y_start <= my_y && my_y < y_end &&
z_start <= my_z && my_z < z_end) {
pos[p_local] = my_x;
pos[p_local + 1] = my_y;
pos[p_local + 2] = my_z;
charges[p_local/3] = my_q;
p_local += 3;
}
}
}
/* create FCSParameter object. IMPORTANT: pp3mg requires a cartesian MPI communicator
which has to be created by the calling program */
fcs_result = fcs_init(&fcs_handle, "pp3mg", mpi_comm_cart);
ASSERT_FCS(fcs_result);
fcs_result = fcs_set_dimension (fcs_handle, DIM);
ASSERT_FCS(fcs_result);
fcs_result = fcs_set_common(fcs_handle, 1,
box[0], box[1], box[2], offset, periodic_flags,
particles);
ASSERT_FCS(fcs_result);
fcs_result = fcs_pp3mg_setup (fcs_handle, mpi_dims, cells_x, cells_y,
cells_z, ghosts, max_iterations, max_particles,
periodic_flags[0] /* FIXME */, degree, err_bound);
ASSERT_FCS(fcs_result);
fcs_result = fcs_tune(fcs_handle, local_particles, pos, charges);
ASSERT_FCS(fcs_result);
MPI_Barrier(mpi_comm_cart);
if(my_rank == 0) {
starttime = MPI_Wtime();
}
/* 2. step: run pp3mg. IMPORTANT: domain decomposition, particles must be distributed */
/* according to the specified MPI communicator */
fcs_result = fcs_run(fcs_handle, local_particles, pos, charges, field, potentials);
ASSERT_FCS(fcs_result);
/* analyze results */
/* field = fcsOutput_getField(fcs_output);
potentials = fcsOutput_getPotentials(fcs_output); */
MPI_Barrier(mpi_comm_cart);
if(my_rank == 0) {
endtime = MPI_Wtime();
printf("pp3mg runtime with %d processors: %f s\n", mpi_size, endtime - starttime);
}
e_sum_local = 0.0;
for(i = 0; i < local_particles; ++i)
e_sum_local += potentials[i];
e_sum = 0.0;
MPI_Reduce(&e_sum_local,
&e_sum,
1,
MPI_DOUBLE,
MPI_SUM,
0,
mpi_comm_cart);
if(my_rank == 0) {
min = cells_x;
if(cells_y < min || cells_z < min)
min = (cells_y < cells_z) ? cells_y : cells_z;
printf("Self energy: %e\n",
1.0/(4.0 * PI) * 14.0/(5.0*1./(2.*ghosts*min)));
printf("Approx. Madelung's constant: %e\n", e_sum/particles *2.0 * PI);
printf( "Total energy:: %e\n\n", e_sum);
}
for(i = 0; i < DIM; ++i) {
f_sum_local[i] = 0.0;
f_sum[i] = 0.0;
f_sum_squared_local[i] = 0.0;
f_sum_squared[i] = 0.0;
f_max_abs_local[i] = 0.0;
f_max_abs[i] = 0.0;
}
for(i = 0; i < local_particles; ++i) {
for(j = 0; j < DIM; ++j) {
f_sum_local[j] += charges[i] * field[i*DIM + j];
f_sum_squared_local[j] += charges[i] * charges[i] * field[i*DIM + j]*field[i*DIM + j];
if(fabs(field[i*DIM + j]) > f_max_abs_local[j]) {
f_max_abs_local[j] = fabs(charges[i] * field[i*DIM + j]);
}/* end if */
}/* end for-j */
}/* end for-i */
for(j = 0; j < DIM; ++j) {
f_max_abs[j] = f_max_abs_local[j];
}
MPI_Reduce(f_sum_local, f_sum, 3, MPI_DOUBLE, MPI_SUM, 0, mpi_comm_cart);
MPI_Reduce(f_sum_squared_local, f_sum_squared, 3,
MPI_DOUBLE, MPI_SUM, 0, mpi_comm_cart);
MPI_Reduce(f_max_abs_local, f_max_abs, 3,
MPI_DOUBLE, MPI_MAX, 0, mpi_comm_cart);
if(my_rank == 0) {
printf("Norm of sum of forces: %e\n",
sqrt(f_sum[0]*f_sum[0] + f_sum[1]*f_sum[1] + f_sum[2]*f_sum[2]) );
printf("Sqrt. of sum of squares of all components of forces: %f\n",
sqrt(f_sum_squared[0] + f_sum_squared[1] + f_sum_squared[2]));
printf("Maximal absolute force components: \n");
for(i = 0; i < DIM; ++i) {
printf("%f ", f_max_abs[i]);
}
printf("\n");
}
if(local_particles != 0)
fclose(input_file);
/* 3. step: deallocate resources for pp3mg */
fcs_destroy(fcs_handle);
free (pos);
free (charges);
free (field);
free (potentials);
MPI_Comm_free(&mpi_comm_cart);
MPI_Finalize();
exit(0);
}
int main(int argc, char **argv)
{
fcs_int total_num_particles = TEST_N_PARTICLES;
fcs_int num_particles, max_num_particles;
fcs_float box_size = TEST_BOX_SIZE;
fcs_int pid, px, py, pz;
fcs_float positions[3*TEST_N_PARTICLES];
fcs_float charges[TEST_N_PARTICLES];
fcs_float direct_fields[3*TEST_N_PARTICLES];
fcs_float direct_potentials[TEST_N_PARTICLES];
fcs_float p2nfft_fields[3*TEST_N_PARTICLES];
fcs_float p2nfft_potentials[TEST_N_PARTICLES];
fcs_float direct_virial[9], p2nfft_virial[9];
MPI_Init(&argc, &argv);
MPI_Comm comm = MPI_COMM_WORLD;
int comm_rank, comm_size;
MPI_Comm_size(comm, &comm_size);
MPI_Comm_rank(comm, &comm_rank);
pid = num_particles = 0;
for (px = 0; px < TEST_BOX_SIZE; ++px) {
for (py = 0; py < TEST_BOX_SIZE; ++py) {
for (pz = 0; pz < TEST_BOX_SIZE; ++pz, ++pid) {
if (pid % comm_size == comm_rank) {
positions[3*num_particles] = px + 0.5;
positions[3*num_particles + 1] = py + 0.5;
positions[3*num_particles + 2] = pz + 0.5;
charges[num_particles] = 1.0-((px + py + pz) % 2)*2;
++num_particles;
}
}
}
}
max_num_particles = num_particles;
/* Debugging */
for(int t=0; t<6; t++)
fprintf(stderr, "init positions[%d] = %" FCS_LMOD_FLOAT "f\n", t, positions[t]);
fcs_float box_a[] = { box_size, 0.0, 0.0 };
fcs_float box_b[] = { 0.0, box_size, 0.0 };
fcs_float box_c[] = { 0.0, 0.0, box_size };
fcs_float offset[] = {0.0, 0.0, 0.0};
fcs_int periodicity[] = {0, 0, 0};
// fcs_int periodicity[] = {1, 1, 1};
FCS fcs_handle = NULL;
FCSResult fcs_result = NULL;
/* Calculate this system via FCS direct solver */
fcs_result = fcs_init(&fcs_handle, "direct", comm);
assert_fcs(fcs_result);
fcs_result = fcs_set_common(fcs_handle, 1, box_a, box_b, box_c, offset, periodicity, total_num_particles);
assert_fcs(fcs_result);
fcs_result = fcs_set_max_local_particles(fcs_handle, max_num_particles);
assert_fcs(fcs_result);
fcs_result = fcs_tune(fcs_handle, num_particles, positions, charges);
assert_fcs(fcs_result);
fcs_result = fcs_set_compute_virial(fcs_handle, 1);
assert_fcs(fcs_result);
/* Debugging */
for(int t=0; t<6; t++)
fprintf(stderr, "before direct run: positions[%d] = %" FCS_LMOD_FLOAT "f\n", t, positions[t]);
fcs_result = fcs_run(fcs_handle, num_particles, positions, charges,
direct_fields, direct_potentials);
assert_fcs(fcs_result);
/* Debugging */
for(int t=0; t<6; t++)
fprintf(stderr, "after direct run: positions[%d] = %" FCS_LMOD_FLOAT "f\n", t, positions[t]);
fcs_result = fcs_get_virial(fcs_handle, direct_virial);
assert_fcs(fcs_result);
printf("Virial via FCS direct:\n");
if(!comm_rank)
printf("virial tensor = [%" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e; %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e; %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e]\n", direct_virial[0], direct_virial[1], direct_virial[2],
direct_virial[3], direct_virial[4], direct_virial[5], direct_virial[6], direct_virial[7], direct_virial[8]);
printf("Potentials via FCS direct:\n");
for (pid = 0; pid < num_particles; ++pid)
printf("%" FCS_LMOD_FLOAT "f\n", direct_potentials[pid]);
printf("Fields via FCS direct:\n");
for (pid = 0; pid < num_particles; ++pid)
printf("[%" FCS_LMOD_FLOAT "f %" FCS_LMOD_FLOAT "f %" FCS_LMOD_FLOAT "f]\n", direct_fields[3*pid+0], direct_fields[3*pid+1], direct_fields[3*pid+2]);
fcs_destroy(fcs_handle);
/* set p2nfft specific parameters */
fcs_float tolerance = 1.e-3;
/* Calculate this system via FCS p2nfft solver */
fcs_result = fcs_init(&fcs_handle, "p2nfft", comm);
assert_fcs(fcs_result);
fcs_result = fcs_set_common(fcs_handle, 1, box_a, box_b, box_c, offset, periodicity, total_num_particles);
assert_fcs(fcs_result);
fcs_result = fcs_set_max_local_particles(fcs_handle, max_num_particles);
assert_fcs(fcs_result);
fcs_set_tolerance(fcs_handle, FCS_TOLERANCE_TYPE_POTENTIAL, tolerance);
fcs_result = fcs_tune(fcs_handle, num_particles, positions, charges);
assert_fcs(fcs_result);
fcs_result = fcs_set_compute_virial(fcs_handle, 1);
assert_fcs(fcs_result);
/* Debugging */
for(int t=0; t<6; t++)
fprintf(stderr, "test: positions[%d] = %" FCS_LMOD_FLOAT "f\n", t, positions[t]);
fcs_result = fcs_run(fcs_handle, num_particles, positions, charges,
p2nfft_fields, p2nfft_potentials);
assert_fcs(fcs_result);
fcs_result = fcs_get_virial(fcs_handle, p2nfft_virial);
assert_fcs(fcs_result);
printf("Virial via FCS p2nfft:\n");
if(!comm_rank)
printf("virial tensor = [%" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e; %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e; %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e %" FCS_LMOD_FLOAT "e]\n", p2nfft_virial[0], p2nfft_virial[1], p2nfft_virial[2],
p2nfft_virial[3], p2nfft_virial[4], p2nfft_virial[5], p2nfft_virial[6], p2nfft_virial[7], p2nfft_virial[8]);
printf("Potentials via FCS p2nfft:\n");
for (pid = 0; pid < num_particles; ++pid)
printf("%" FCS_LMOD_FLOAT "f\n", p2nfft_potentials[pid]);
printf("Fields via FCS p2nfft:\n");
for (pid = 0; pid < num_particles; ++pid)
printf("[%" FCS_LMOD_FLOAT "f %" FCS_LMOD_FLOAT "f %" FCS_LMOD_FLOAT "f]\n", p2nfft_fields[3*pid+0], p2nfft_fields[3*pid+1], p2nfft_fields[3*pid+2]);
fcs_destroy(fcs_handle);
/* Compare results of direct and p2nfft solver */
fcs_float direct_energy = 0, p2nfft_energy = 0;
for (pid = 0; pid < num_particles; pid++) {
direct_energy += 0.5 * charges[pid] * direct_potentials[pid];
p2nfft_energy += 0.5 * charges[pid] * p2nfft_potentials[pid];
}
fcs_float sqr_sum = 0.0;
for (pid = 0; pid < num_particles; ++pid) {
fcs_float d0 = p2nfft_fields[3*pid] - direct_fields[3*pid];
fcs_float d1 = p2nfft_fields[3*pid+1] - direct_fields[3*pid+1];
fcs_float d2 = p2nfft_fields[3*pid+2] - direct_fields[3*pid+2];
sqr_sum += d0*d0+d1*d1+d2*d2;
}
/* Reduce to global values */
fcs_float direct_total_energy, p2nfft_total_energy, total_sqr_sum;
MPI_Reduce(&direct_energy, &direct_total_energy, 1, FCS_MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&p2nfft_energy, &p2nfft_total_energy, 1, FCS_MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&sqr_sum, &total_sqr_sum, 1, FCS_MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
if (!comm_rank) {
printf("direct_energy = %" FCS_LMOD_FLOAT "f\n", direct_total_energy);
printf("p2nfft_energy = %" FCS_LMOD_FLOAT "f\n", p2nfft_total_energy);
printf("rms_error = %e\n", sqrt(total_sqr_sum / (fcs_float)total_num_particles));
}
MPI_Finalize();
return 0;
}