void
sc_abort_collective (const char *msg)
{
int mpiret;
if (sc_mpicomm != sc_MPI_COMM_NULL) {
mpiret = sc_MPI_Barrier (sc_mpicomm);
SC_CHECK_MPI (mpiret);
}
if (sc_is_root ()) {
SC_ABORT (msg);
}
else {
sleep (3); /* wait for root rank's sc_MPI_Abort ()... */
abort (); /* ... otherwise this may call sc_MPI_Abort () */
}
}
static void
run_load (sc_MPI_Comm mpicomm, p4est_connectivity_t * conn, int level)
{
int mpiret;
double elapsed_create, elapsed_partition, elapsed_balance;
#ifdef LOADCONN_VTK
char filename[BUFSIZ];
#endif
p4est_t *p4est;
P4EST_GLOBAL_PRODUCTIONF ("Run load on level %d\n", level);
/* create and refine the forest */
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
elapsed_create = -sc_MPI_Wtime ();
p4est = p4est_new_ext (mpicomm, conn, 0, level, 1, 0, NULL, NULL);
level_shift = 4;
refine_level = level + level_shift;
p4est_refine (p4est, 1, refine_fractal, NULL);
elapsed_create += sc_MPI_Wtime ();
#ifdef LOADCONN_VTK
snprintf (filename, BUFSIZ, "loadconn%d_%02d_C", P4EST_DIM, level);
p4est_vtk_write_file (p4est, NULL, filename);
#endif
/* partition the forest */
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
elapsed_partition = -sc_MPI_Wtime ();
p4est_partition (p4est, 0, NULL);
elapsed_partition += sc_MPI_Wtime ();
/* balance the forest */
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
elapsed_balance = -sc_MPI_Wtime ();
p4est_balance (p4est, P4EST_CONNECT_FULL, NULL);
elapsed_balance += sc_MPI_Wtime ();
#ifdef LOADCONN_VTK
snprintf (filename, BUFSIZ, "loadconn%d_%02d_B", P4EST_DIM, level);
p4est_vtk_write_file (p4est, NULL, filename);
#endif
/* report timings */
P4EST_GLOBAL_PRODUCTIONF ("Timings %d: %g %g %g\n", level, elapsed_create,
elapsed_partition, elapsed_balance);
p4est_destroy (p4est);
}
static void
test_loadsave (p4est_connectivity_t * connectivity, const char *prefix,
sc_MPI_Comm mpicomm, int mpirank)
{
int mpiret, retval;
unsigned csum, csum2;
double elapsed, wtime;
p4est_connectivity_t *conn2;
p4est_t *p4est, *p4est2;
sc_statinfo_t stats[STATS_COUNT];
char conn_name[BUFSIZ];
char p4est_name[BUFSIZ];
snprintf (conn_name, BUFSIZ, "%s.%s", prefix, P4EST_CONN_SUFFIX);
snprintf (p4est_name, BUFSIZ, "%s.%s", prefix, P4EST_FOREST_SUFFIX);
P4EST_GLOBAL_INFOF ("Using file names %s and %s\n", conn_name, p4est_name);
p4est = p4est_new_ext (mpicomm, connectivity, 0, 0, 0,
sizeof (int), init_fn, NULL);
p4est_refine (p4est, 1, refine_fn, init_fn);
test_deflate (p4est);
/* save, synchronize, load connectivity and compare */
if (mpirank == 0) {
retval = p4est_connectivity_save (conn_name, connectivity);
SC_CHECK_ABORT (retval == 0, "connectivity_save failed");
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
wtime = sc_MPI_Wtime ();
conn2 = p4est_connectivity_load (conn_name, NULL);
elapsed = sc_MPI_Wtime () - wtime;
sc_stats_set1 (stats + STATS_CONN_LOAD, elapsed, "conn load");
SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2),
"load/save connectivity mismatch A");
p4est_connectivity_destroy (conn2);
/* save, synchronize, load p4est and compare */
wtime = sc_MPI_Wtime ();
p4est_save (p4est_name, p4est, 1);
elapsed = sc_MPI_Wtime () - wtime;
sc_stats_set1 (stats + STATS_P4EST_SAVE1, elapsed, "p4est save 1");
wtime = sc_MPI_Wtime ();
p4est2 = p4est_load (p4est_name, mpicomm, sizeof (int), 1, NULL, &conn2);
elapsed = sc_MPI_Wtime () - wtime;
sc_stats_set1 (stats + STATS_P4EST_LOAD1a, elapsed, "p4est load 1a");
SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2),
"load/save connectivity mismatch Ba");
SC_CHECK_ABORT (p4est_is_equal (p4est, p4est2, 1),
"load/save p4est mismatch Ba");
p4est_destroy (p4est2);
p4est_connectivity_destroy (conn2);
wtime = sc_MPI_Wtime ();
p4est2 = p4est_load (p4est_name, mpicomm, 0, 0, NULL, &conn2);
elapsed = sc_MPI_Wtime () - wtime;
sc_stats_set1 (stats + STATS_P4EST_LOAD1b, elapsed, "p4est load 1b");
SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2),
"load/save connectivity mismatch Bb");
SC_CHECK_ABORT (p4est_is_equal (p4est, p4est2, 0),
"load/save p4est mismatch Bb");
test_deflate (p4est2);
p4est_destroy (p4est2);
p4est_connectivity_destroy (conn2);
/* partition and balance */
p4est_partition (p4est, 0, NULL);
p4est_balance (p4est, P4EST_CONNECT_FULL, init_fn);
csum = p4est_checksum (p4est);
sc_stats_set1 (stats + STATS_P4EST_ELEMS,
(double) p4est->local_num_quadrants, "p4est elements");
/* save, synchronize, load p4est and compare */
wtime = sc_MPI_Wtime ();
p4est_save (p4est_name, p4est, 0);
elapsed = sc_MPI_Wtime () - wtime;
sc_stats_set1 (stats + STATS_P4EST_SAVE2, elapsed, "p4est save 2");
wtime = sc_MPI_Wtime ();
p4est2 = p4est_load (p4est_name, mpicomm, sizeof (int), 0, NULL, &conn2);
elapsed = sc_MPI_Wtime () - wtime;
sc_stats_set1 (stats + STATS_P4EST_LOAD2, elapsed, "p4est load 2");
SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2),
"load/save connectivity mismatch C");
SC_CHECK_ABORT (p4est_is_equal (p4est, p4est2, 0),
"load/save p4est mismatch C");
p4est_destroy (p4est2);
p4est_connectivity_destroy (conn2);
/* save, synchronize, load p4est and compare */
wtime = sc_MPI_Wtime ();
p4est_save (p4est_name, p4est, 1);
elapsed = sc_MPI_Wtime () - wtime;
sc_stats_set1 (stats + STATS_P4EST_SAVE3, elapsed, "p4est save 3");
wtime = sc_MPI_Wtime ();
p4est2 = p4est_load (p4est_name, mpicomm, sizeof (int), 0, NULL, &conn2);
elapsed = sc_MPI_Wtime () - wtime;
sc_stats_set1 (stats + STATS_P4EST_LOAD3, elapsed, "p4est load 3");
SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2),
"load/save connectivity mismatch D");
SC_CHECK_ABORT (p4est_is_equal (p4est, p4est2, 0),
"load/save p4est mismatch D");
p4est_destroy (p4est2);
p4est_connectivity_destroy (conn2);
/* Test autopartition load feature */
wtime = sc_MPI_Wtime ();
p4est2 = p4est_load_ext (p4est_name, mpicomm, sizeof (int), 0,
1, 0, NULL, &conn2);
elapsed = sc_MPI_Wtime () - wtime;
csum2 = p4est_checksum (p4est2);
sc_stats_set1 (stats + STATS_P4EST_LOAD4, elapsed, "p4est load 4");
SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2),
"load/save connectivity mismatch E");
SC_CHECK_ABORT (mpirank != 0 || csum == csum2,
"load/save p4est mismatch E");
p4est_destroy (p4est2);
p4est_connectivity_destroy (conn2);
/* destroy data structures */
p4est_destroy (p4est);
/* compute and print timings */
sc_stats_compute (mpicomm, STATS_COUNT, stats);
sc_stats_print (p4est_package_id, SC_LP_STATISTICS,
STATS_COUNT, stats, 0, 1);
}
/**
* Runs all tests.
*/
int
main (int argc, char **argv)
{
const char *this_fn_name = P4EST_STRING "_test_subcomm";
/* options */
const p4est_locidx_t min_quadrants = 15;
const int min_level = 4;
const int fill_uniform = 0;
/* parallel environment */
sc_MPI_Comm mpicomm = sc_MPI_COMM_WORLD;
int mpisize, submpisize;
int mpiret;
#ifdef P4EST_ENABLE_DEBUG
int rank;
#endif
/* p4est */
p4est_connectivity_t *connectivity;
p4est_t *p4est;
p4est_locidx_t *partition;
/* initialize MPI */
mpiret = sc_MPI_Init (&argc, &argv);
SC_CHECK_MPI (mpiret);
/* exit if MPI communicator cannot be reduced */
mpiret = sc_MPI_Comm_size (mpicomm, &mpisize);
SC_CHECK_MPI (mpiret);
if (mpisize == 1) {
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
/* initialize p4est */
sc_init (mpicomm, 1, 1, NULL, SC_LP_DEFAULT);
p4est_init (NULL, SC_LP_DEFAULT);
/* create connectivity */
#ifdef P4_TO_P8
connectivity = p8est_connectivity_new_unitcube ();
#else
connectivity = p4est_connectivity_new_unitsquare ();
#endif
/* create p4est object */
p4est = p4est_new_ext (mpicomm, connectivity,
min_quadrants, min_level, fill_uniform,
0, NULL, NULL);
/* write vtk: new */
p4est_vtk_write_file (p4est, NULL, P4EST_STRING "_subcomm_new");
/* set variables pertaining to the parallel environment */
#ifdef P4EST_ENABLE_DEBUG
rank = p4est->mpirank;
#endif
submpisize = mpisize / 2;
P4EST_ASSERT (submpisize <= p4est->global_num_quadrants);
/* construct partitioning with empty ranks */
{
p4est_locidx_t n_quads_per_proc, n_quads_leftover;
int p;
partition = P4EST_ALLOC (p4est_locidx_t, mpisize);
n_quads_per_proc = p4est->global_num_quadrants / submpisize;
n_quads_leftover = p4est->global_num_quadrants -
(n_quads_per_proc * submpisize);
for (p = 0; p < mpisize; p++) {
if (p % 2) { /* if this rank will get quadrants */
partition[p] = n_quads_per_proc;
}
else { /* if this rank will be empty */
partition[p] = 0;
}
}
partition[1] += n_quads_leftover;
/* check partitioning */
#ifdef P4EST_ENABLE_DEBUG
{
p4est_gloidx_t sum = 0;
for (p = 0; p < mpisize; p++) {
sum += (p4est_gloidx_t) partition[p];
}
P4EST_ASSERT (sum == p4est->global_num_quadrants);
}
#endif
}
/*
* Test 1: Reduce MPI communicator to non-empty ranks
*/
P4EST_GLOBAL_INFOF ("%s: Into test 1\n", this_fn_name);
{
p4est_t *p4est_subcomm;
int is_nonempty;
/* create p4est copy and re-partition */
p4est_subcomm = p4est_copy_ext (p4est, 1, 1);
(void) p4est_partition_given (p4est_subcomm, partition);
/* write vtk: partitioned */
p4est_vtk_write_file (p4est_subcomm, NULL, P4EST_STRING "_subcomm_part");
/* reduce MPI communicator to non-empty ranks */
is_nonempty = p4est_comm_parallel_env_reduce (&p4est_subcomm);
P4EST_ASSERT ((is_nonempty && 0 < partition[rank]) ||
(!is_nonempty && 0 == partition[rank]));
if (is_nonempty) {
/* write vtk: reduced communicator */
p4est_vtk_write_file (p4est_subcomm, NULL,
P4EST_STRING "_subcomm_sub1");
/* destroy the p4est that has a reduced MPI communicator */
p4est_destroy (p4est_subcomm);
}
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_INFOF ("%s: Done test 1\n", this_fn_name);
/*
* Test 2: Reduce MPI communicator to non-empty ranks, but now the MPI
* communicator is not owned
*/
P4EST_GLOBAL_INFOF ("%s: Into test 2\n", this_fn_name);
{
p4est_t *p4est_subcomm;
int is_nonempty;
/* create p4est copy and re-partition */
p4est_subcomm = p4est_copy_ext (p4est, 1, 0 /* don't dup. comm. */ );
(void) p4est_partition_given (p4est_subcomm, partition);
/* reduce MPI communicator to non-empty ranks */
is_nonempty = p4est_comm_parallel_env_reduce (&p4est_subcomm);
P4EST_ASSERT ((is_nonempty && 0 < partition[rank]) ||
(!is_nonempty && 0 == partition[rank]));
if (is_nonempty) {
/* destroy the p4est that has a reduced MPI communicator */
p4est_destroy (p4est_subcomm);
}
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_INFOF ("%s: Done test 2\n", this_fn_name);
/*
* Test 3: Reduce MPI communicator to non-empty ranks, but keep rank 0
*/
P4EST_GLOBAL_INFOF ("%s: Into test 3\n", this_fn_name);
{
p4est_t *p4est_subcomm;
int sub_exists;
sc_MPI_Group group, group_reserve;
int reserve_range[1][3];
/* create group of full MPI communicator */
mpiret = sc_MPI_Comm_group (mpicomm, &group);
SC_CHECK_MPI (mpiret);
/* create sub-group containing only rank 0 */
reserve_range[0][0] = 0;
reserve_range[0][1] = 0;
reserve_range[0][2] = 1;
mpiret =
sc_MPI_Group_range_incl (group, 1, reserve_range, &group_reserve);
SC_CHECK_MPI (mpiret);
/* create p4est copy and re-partition */
p4est_subcomm = p4est_copy_ext (p4est, 1, 1);
(void) p4est_partition_given (p4est_subcomm, partition);
/* reduce MPI communicator to non-empty ranks, but keep rank 0 */
sub_exists = p4est_comm_parallel_env_reduce_ext (&p4est_subcomm,
group_reserve, 1, NULL);
P4EST_ASSERT ((sub_exists && (0 < partition[rank] || rank == 0)) ||
(!sub_exists && 0 == partition[rank]));
if (sub_exists) {
/* write vtk: reduced communicator */
p4est_vtk_write_file (p4est_subcomm, NULL,
P4EST_STRING "_subcomm_sub3");
/* destroy the p4est that has a reduced MPI communicator */
p4est_destroy (p4est_subcomm);
}
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_INFOF ("%s: Done test 3\n", this_fn_name);
/*
* Test 4: Reduce MPI communicator to non-empty ranks, but keep last 2 ranks
*/
P4EST_GLOBAL_INFOF ("%s: Into test 4\n", this_fn_name);
{
p4est_t *p4est_subcomm;
int sub_exists;
sc_MPI_Group group, group_reserve;
int reserve_range[1][3];
/* create group of full MPI communicator */
mpiret = sc_MPI_Comm_group (mpicomm, &group);
SC_CHECK_MPI (mpiret);
/* create sub-group containing only last 2 ranks */
reserve_range[0][0] = SC_MAX (0, mpisize - 2);
reserve_range[0][1] = mpisize - 1;
reserve_range[0][2] = 1;
mpiret =
sc_MPI_Group_range_incl (group, 1, reserve_range, &group_reserve);
SC_CHECK_MPI (mpiret);
/* create p4est copy and re-partition */
p4est_subcomm = p4est_copy_ext (p4est, 1, 1);
(void) p4est_partition_given (p4est_subcomm, partition);
/* reduce MPI communicator to non-empty ranks, but keep last 2 ranks */
sub_exists = p4est_comm_parallel_env_reduce_ext (&p4est_subcomm,
group_reserve, 0, NULL);
P4EST_ASSERT ((sub_exists && (0 < partition[rank] || mpisize - 2 <= rank))
|| (!sub_exists && 0 == partition[rank]));
if (sub_exists) {
/* write vtk: reduced communicator */
p4est_vtk_write_file (p4est_subcomm, NULL,
P4EST_STRING "_subcomm_sub4");
/* destroy the p4est that has a reduced MPI communicator */
p4est_destroy (p4est_subcomm);
}
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_INFOF ("%s: Done test 4\n", this_fn_name);
/* destroy */
P4EST_FREE (partition);
p4est_destroy (p4est);
p4est_connectivity_destroy (connectivity);
/* finalize */
sc_finalize ();
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
static void
run_bricks (sc_MPI_Comm mpicomm, int per, int l, int rlevel)
{
int mpiret;
int tcount;
double elapsed_create, elapsed_partition, elapsed_balance;
#ifdef BRICKS_VTK
char filename[BUFSIZ];
#endif
p4est_connectivity_t *conn;
p4est_t *p4est;
P4EST_GLOBAL_PRODUCTIONF ("Run bricks on level %d/%d\n", l, rlevel);
P4EST_ASSERT (l <= rlevel);
/* create and refine the forest */
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
elapsed_create = -sc_MPI_Wtime ();
tcount = 1 << l;
#ifndef P4_TO_P8
conn = p4est_connectivity_new_brick (tcount, tcount, per, per);
#else
conn = p8est_connectivity_new_brick (tcount, tcount, tcount, per, per, per);
#endif
p4est = p4est_new_ext (mpicomm, conn, 0, rlevel - l, 1, 0, NULL, NULL);
level_shift = 4;
refine_level = rlevel - l + level_shift;
p4est_refine (p4est, 1, refine_fractal, NULL);
elapsed_create += sc_MPI_Wtime ();
/* partition the forest */
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
elapsed_partition = -sc_MPI_Wtime ();
p4est_partition (p4est, 0, NULL);
elapsed_partition += sc_MPI_Wtime ();
/* balance the forest */
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
elapsed_balance = -sc_MPI_Wtime ();
p4est_balance (p4est, P4EST_CONNECT_FULL, NULL);
elapsed_balance += sc_MPI_Wtime ();
/* postprocessing */
P4EST_GLOBAL_PRODUCTIONF ("Timings %g %g %g\n", elapsed_create,
elapsed_partition, elapsed_balance);
#ifdef BRICKS_VTK
snprintf (filename, BUFSIZ, "brick_%02d_%02d_B", rlevel, l);
p4est_vtk_write_file (p4est, NULL, filename);
#endif
p4est_destroy (p4est);
p4est_connectivity_destroy (conn);
}
