void sc_abort_collective (const char *msg) { int mpiret; if (sc_mpicomm != sc_MPI_COMM_NULL) { mpiret = sc_MPI_Barrier (sc_mpicomm); SC_CHECK_MPI (mpiret); } if (sc_is_root ()) { SC_ABORT (msg); } else { sleep (3); /* wait for root rank's sc_MPI_Abort ()... */ abort (); /* ... otherwise this may call sc_MPI_Abort () */ } }
static void run_load (sc_MPI_Comm mpicomm, p4est_connectivity_t * conn, int level) { int mpiret; double elapsed_create, elapsed_partition, elapsed_balance; #ifdef LOADCONN_VTK char filename[BUFSIZ]; #endif p4est_t *p4est; P4EST_GLOBAL_PRODUCTIONF ("Run load on level %d\n", level); /* create and refine the forest */ mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); elapsed_create = -sc_MPI_Wtime (); p4est = p4est_new_ext (mpicomm, conn, 0, level, 1, 0, NULL, NULL); level_shift = 4; refine_level = level + level_shift; p4est_refine (p4est, 1, refine_fractal, NULL); elapsed_create += sc_MPI_Wtime (); #ifdef LOADCONN_VTK snprintf (filename, BUFSIZ, "loadconn%d_%02d_C", P4EST_DIM, level); p4est_vtk_write_file (p4est, NULL, filename); #endif /* partition the forest */ mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); elapsed_partition = -sc_MPI_Wtime (); p4est_partition (p4est, 0, NULL); elapsed_partition += sc_MPI_Wtime (); /* balance the forest */ mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); elapsed_balance = -sc_MPI_Wtime (); p4est_balance (p4est, P4EST_CONNECT_FULL, NULL); elapsed_balance += sc_MPI_Wtime (); #ifdef LOADCONN_VTK snprintf (filename, BUFSIZ, "loadconn%d_%02d_B", P4EST_DIM, level); p4est_vtk_write_file (p4est, NULL, filename); #endif /* report timings */ P4EST_GLOBAL_PRODUCTIONF ("Timings %d: %g %g %g\n", level, elapsed_create, elapsed_partition, elapsed_balance); p4est_destroy (p4est); }
static void test_loadsave (p4est_connectivity_t * connectivity, const char *prefix, sc_MPI_Comm mpicomm, int mpirank) { int mpiret, retval; unsigned csum, csum2; double elapsed, wtime; p4est_connectivity_t *conn2; p4est_t *p4est, *p4est2; sc_statinfo_t stats[STATS_COUNT]; char conn_name[BUFSIZ]; char p4est_name[BUFSIZ]; snprintf (conn_name, BUFSIZ, "%s.%s", prefix, P4EST_CONN_SUFFIX); snprintf (p4est_name, BUFSIZ, "%s.%s", prefix, P4EST_FOREST_SUFFIX); P4EST_GLOBAL_INFOF ("Using file names %s and %s\n", conn_name, p4est_name); p4est = p4est_new_ext (mpicomm, connectivity, 0, 0, 0, sizeof (int), init_fn, NULL); p4est_refine (p4est, 1, refine_fn, init_fn); test_deflate (p4est); /* save, synchronize, load connectivity and compare */ if (mpirank == 0) { retval = p4est_connectivity_save (conn_name, connectivity); SC_CHECK_ABORT (retval == 0, "connectivity_save failed"); } mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); wtime = sc_MPI_Wtime (); conn2 = p4est_connectivity_load (conn_name, NULL); elapsed = sc_MPI_Wtime () - wtime; sc_stats_set1 (stats + STATS_CONN_LOAD, elapsed, "conn load"); SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2), "load/save connectivity mismatch A"); p4est_connectivity_destroy (conn2); /* save, synchronize, load p4est and compare */ wtime = sc_MPI_Wtime (); p4est_save (p4est_name, p4est, 1); elapsed = sc_MPI_Wtime () - wtime; sc_stats_set1 (stats + STATS_P4EST_SAVE1, elapsed, "p4est save 1"); wtime = sc_MPI_Wtime (); p4est2 = p4est_load (p4est_name, mpicomm, sizeof (int), 1, NULL, &conn2); elapsed = sc_MPI_Wtime () - wtime; sc_stats_set1 (stats + STATS_P4EST_LOAD1a, elapsed, "p4est load 1a"); SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2), "load/save connectivity mismatch Ba"); SC_CHECK_ABORT (p4est_is_equal (p4est, p4est2, 1), "load/save p4est mismatch Ba"); p4est_destroy (p4est2); p4est_connectivity_destroy (conn2); wtime = sc_MPI_Wtime (); p4est2 = p4est_load (p4est_name, mpicomm, 0, 0, NULL, &conn2); elapsed = sc_MPI_Wtime () - wtime; sc_stats_set1 (stats + STATS_P4EST_LOAD1b, elapsed, "p4est load 1b"); SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2), "load/save connectivity mismatch Bb"); SC_CHECK_ABORT (p4est_is_equal (p4est, p4est2, 0), "load/save p4est mismatch Bb"); test_deflate (p4est2); p4est_destroy (p4est2); p4est_connectivity_destroy (conn2); /* partition and balance */ p4est_partition (p4est, 0, NULL); p4est_balance (p4est, P4EST_CONNECT_FULL, init_fn); csum = p4est_checksum (p4est); sc_stats_set1 (stats + STATS_P4EST_ELEMS, (double) p4est->local_num_quadrants, "p4est elements"); /* save, synchronize, load p4est and compare */ wtime = sc_MPI_Wtime (); p4est_save (p4est_name, p4est, 0); elapsed = sc_MPI_Wtime () - wtime; sc_stats_set1 (stats + STATS_P4EST_SAVE2, elapsed, "p4est save 2"); wtime = sc_MPI_Wtime (); p4est2 = p4est_load (p4est_name, mpicomm, sizeof (int), 0, NULL, &conn2); elapsed = sc_MPI_Wtime () - wtime; sc_stats_set1 (stats + STATS_P4EST_LOAD2, elapsed, "p4est load 2"); SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2), "load/save connectivity mismatch C"); SC_CHECK_ABORT (p4est_is_equal (p4est, p4est2, 0), "load/save p4est mismatch C"); p4est_destroy (p4est2); p4est_connectivity_destroy (conn2); /* save, synchronize, load p4est and compare */ wtime = sc_MPI_Wtime (); p4est_save (p4est_name, p4est, 1); elapsed = sc_MPI_Wtime () - wtime; sc_stats_set1 (stats + STATS_P4EST_SAVE3, elapsed, "p4est save 3"); wtime = sc_MPI_Wtime (); p4est2 = p4est_load (p4est_name, mpicomm, sizeof (int), 0, NULL, &conn2); elapsed = sc_MPI_Wtime () - wtime; sc_stats_set1 (stats + STATS_P4EST_LOAD3, elapsed, "p4est load 3"); SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2), "load/save connectivity mismatch D"); SC_CHECK_ABORT (p4est_is_equal (p4est, p4est2, 0), "load/save p4est mismatch D"); p4est_destroy (p4est2); p4est_connectivity_destroy (conn2); /* Test autopartition load feature */ wtime = sc_MPI_Wtime (); p4est2 = p4est_load_ext (p4est_name, mpicomm, sizeof (int), 0, 1, 0, NULL, &conn2); elapsed = sc_MPI_Wtime () - wtime; csum2 = p4est_checksum (p4est2); sc_stats_set1 (stats + STATS_P4EST_LOAD4, elapsed, "p4est load 4"); SC_CHECK_ABORT (p4est_connectivity_is_equal (connectivity, conn2), "load/save connectivity mismatch E"); SC_CHECK_ABORT (mpirank != 0 || csum == csum2, "load/save p4est mismatch E"); p4est_destroy (p4est2); p4est_connectivity_destroy (conn2); /* destroy data structures */ p4est_destroy (p4est); /* compute and print timings */ sc_stats_compute (mpicomm, STATS_COUNT, stats); sc_stats_print (p4est_package_id, SC_LP_STATISTICS, STATS_COUNT, stats, 0, 1); }
/** * Runs all tests. */ int main (int argc, char **argv) { const char *this_fn_name = P4EST_STRING "_test_subcomm"; /* options */ const p4est_locidx_t min_quadrants = 15; const int min_level = 4; const int fill_uniform = 0; /* parallel environment */ sc_MPI_Comm mpicomm = sc_MPI_COMM_WORLD; int mpisize, submpisize; int mpiret; #ifdef P4EST_ENABLE_DEBUG int rank; #endif /* p4est */ p4est_connectivity_t *connectivity; p4est_t *p4est; p4est_locidx_t *partition; /* initialize MPI */ mpiret = sc_MPI_Init (&argc, &argv); SC_CHECK_MPI (mpiret); /* exit if MPI communicator cannot be reduced */ mpiret = sc_MPI_Comm_size (mpicomm, &mpisize); SC_CHECK_MPI (mpiret); if (mpisize == 1) { mpiret = sc_MPI_Finalize (); SC_CHECK_MPI (mpiret); return 0; } /* initialize p4est */ sc_init (mpicomm, 1, 1, NULL, SC_LP_DEFAULT); p4est_init (NULL, SC_LP_DEFAULT); /* create connectivity */ #ifdef P4_TO_P8 connectivity = p8est_connectivity_new_unitcube (); #else connectivity = p4est_connectivity_new_unitsquare (); #endif /* create p4est object */ p4est = p4est_new_ext (mpicomm, connectivity, min_quadrants, min_level, fill_uniform, 0, NULL, NULL); /* write vtk: new */ p4est_vtk_write_file (p4est, NULL, P4EST_STRING "_subcomm_new"); /* set variables pertaining to the parallel environment */ #ifdef P4EST_ENABLE_DEBUG rank = p4est->mpirank; #endif submpisize = mpisize / 2; P4EST_ASSERT (submpisize <= p4est->global_num_quadrants); /* construct partitioning with empty ranks */ { p4est_locidx_t n_quads_per_proc, n_quads_leftover; int p; partition = P4EST_ALLOC (p4est_locidx_t, mpisize); n_quads_per_proc = p4est->global_num_quadrants / submpisize; n_quads_leftover = p4est->global_num_quadrants - (n_quads_per_proc * submpisize); for (p = 0; p < mpisize; p++) { if (p % 2) { /* if this rank will get quadrants */ partition[p] = n_quads_per_proc; } else { /* if this rank will be empty */ partition[p] = 0; } } partition[1] += n_quads_leftover; /* check partitioning */ #ifdef P4EST_ENABLE_DEBUG { p4est_gloidx_t sum = 0; for (p = 0; p < mpisize; p++) { sum += (p4est_gloidx_t) partition[p]; } P4EST_ASSERT (sum == p4est->global_num_quadrants); } #endif } /* * Test 1: Reduce MPI communicator to non-empty ranks */ P4EST_GLOBAL_INFOF ("%s: Into test 1\n", this_fn_name); { p4est_t *p4est_subcomm; int is_nonempty; /* create p4est copy and re-partition */ p4est_subcomm = p4est_copy_ext (p4est, 1, 1); (void) p4est_partition_given (p4est_subcomm, partition); /* write vtk: partitioned */ p4est_vtk_write_file (p4est_subcomm, NULL, P4EST_STRING "_subcomm_part"); /* reduce MPI communicator to non-empty ranks */ is_nonempty = p4est_comm_parallel_env_reduce (&p4est_subcomm); P4EST_ASSERT ((is_nonempty && 0 < partition[rank]) || (!is_nonempty && 0 == partition[rank])); if (is_nonempty) { /* write vtk: reduced communicator */ p4est_vtk_write_file (p4est_subcomm, NULL, P4EST_STRING "_subcomm_sub1"); /* destroy the p4est that has a reduced MPI communicator */ p4est_destroy (p4est_subcomm); } } mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); P4EST_GLOBAL_INFOF ("%s: Done test 1\n", this_fn_name); /* * Test 2: Reduce MPI communicator to non-empty ranks, but now the MPI * communicator is not owned */ P4EST_GLOBAL_INFOF ("%s: Into test 2\n", this_fn_name); { p4est_t *p4est_subcomm; int is_nonempty; /* create p4est copy and re-partition */ p4est_subcomm = p4est_copy_ext (p4est, 1, 0 /* don't dup. comm. */ ); (void) p4est_partition_given (p4est_subcomm, partition); /* reduce MPI communicator to non-empty ranks */ is_nonempty = p4est_comm_parallel_env_reduce (&p4est_subcomm); P4EST_ASSERT ((is_nonempty && 0 < partition[rank]) || (!is_nonempty && 0 == partition[rank])); if (is_nonempty) { /* destroy the p4est that has a reduced MPI communicator */ p4est_destroy (p4est_subcomm); } } mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); P4EST_GLOBAL_INFOF ("%s: Done test 2\n", this_fn_name); /* * Test 3: Reduce MPI communicator to non-empty ranks, but keep rank 0 */ P4EST_GLOBAL_INFOF ("%s: Into test 3\n", this_fn_name); { p4est_t *p4est_subcomm; int sub_exists; sc_MPI_Group group, group_reserve; int reserve_range[1][3]; /* create group of full MPI communicator */ mpiret = sc_MPI_Comm_group (mpicomm, &group); SC_CHECK_MPI (mpiret); /* create sub-group containing only rank 0 */ reserve_range[0][0] = 0; reserve_range[0][1] = 0; reserve_range[0][2] = 1; mpiret = sc_MPI_Group_range_incl (group, 1, reserve_range, &group_reserve); SC_CHECK_MPI (mpiret); /* create p4est copy and re-partition */ p4est_subcomm = p4est_copy_ext (p4est, 1, 1); (void) p4est_partition_given (p4est_subcomm, partition); /* reduce MPI communicator to non-empty ranks, but keep rank 0 */ sub_exists = p4est_comm_parallel_env_reduce_ext (&p4est_subcomm, group_reserve, 1, NULL); P4EST_ASSERT ((sub_exists && (0 < partition[rank] || rank == 0)) || (!sub_exists && 0 == partition[rank])); if (sub_exists) { /* write vtk: reduced communicator */ p4est_vtk_write_file (p4est_subcomm, NULL, P4EST_STRING "_subcomm_sub3"); /* destroy the p4est that has a reduced MPI communicator */ p4est_destroy (p4est_subcomm); } } mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); P4EST_GLOBAL_INFOF ("%s: Done test 3\n", this_fn_name); /* * Test 4: Reduce MPI communicator to non-empty ranks, but keep last 2 ranks */ P4EST_GLOBAL_INFOF ("%s: Into test 4\n", this_fn_name); { p4est_t *p4est_subcomm; int sub_exists; sc_MPI_Group group, group_reserve; int reserve_range[1][3]; /* create group of full MPI communicator */ mpiret = sc_MPI_Comm_group (mpicomm, &group); SC_CHECK_MPI (mpiret); /* create sub-group containing only last 2 ranks */ reserve_range[0][0] = SC_MAX (0, mpisize - 2); reserve_range[0][1] = mpisize - 1; reserve_range[0][2] = 1; mpiret = sc_MPI_Group_range_incl (group, 1, reserve_range, &group_reserve); SC_CHECK_MPI (mpiret); /* create p4est copy and re-partition */ p4est_subcomm = p4est_copy_ext (p4est, 1, 1); (void) p4est_partition_given (p4est_subcomm, partition); /* reduce MPI communicator to non-empty ranks, but keep last 2 ranks */ sub_exists = p4est_comm_parallel_env_reduce_ext (&p4est_subcomm, group_reserve, 0, NULL); P4EST_ASSERT ((sub_exists && (0 < partition[rank] || mpisize - 2 <= rank)) || (!sub_exists && 0 == partition[rank])); if (sub_exists) { /* write vtk: reduced communicator */ p4est_vtk_write_file (p4est_subcomm, NULL, P4EST_STRING "_subcomm_sub4"); /* destroy the p4est that has a reduced MPI communicator */ p4est_destroy (p4est_subcomm); } } mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); P4EST_GLOBAL_INFOF ("%s: Done test 4\n", this_fn_name); /* destroy */ P4EST_FREE (partition); p4est_destroy (p4est); p4est_connectivity_destroy (connectivity); /* finalize */ sc_finalize (); mpiret = sc_MPI_Finalize (); SC_CHECK_MPI (mpiret); return 0; }
static void run_bricks (sc_MPI_Comm mpicomm, int per, int l, int rlevel) { int mpiret; int tcount; double elapsed_create, elapsed_partition, elapsed_balance; #ifdef BRICKS_VTK char filename[BUFSIZ]; #endif p4est_connectivity_t *conn; p4est_t *p4est; P4EST_GLOBAL_PRODUCTIONF ("Run bricks on level %d/%d\n", l, rlevel); P4EST_ASSERT (l <= rlevel); /* create and refine the forest */ mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); elapsed_create = -sc_MPI_Wtime (); tcount = 1 << l; #ifndef P4_TO_P8 conn = p4est_connectivity_new_brick (tcount, tcount, per, per); #else conn = p8est_connectivity_new_brick (tcount, tcount, tcount, per, per, per); #endif p4est = p4est_new_ext (mpicomm, conn, 0, rlevel - l, 1, 0, NULL, NULL); level_shift = 4; refine_level = rlevel - l + level_shift; p4est_refine (p4est, 1, refine_fractal, NULL); elapsed_create += sc_MPI_Wtime (); /* partition the forest */ mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); elapsed_partition = -sc_MPI_Wtime (); p4est_partition (p4est, 0, NULL); elapsed_partition += sc_MPI_Wtime (); /* balance the forest */ mpiret = sc_MPI_Barrier (mpicomm); SC_CHECK_MPI (mpiret); elapsed_balance = -sc_MPI_Wtime (); p4est_balance (p4est, P4EST_CONNECT_FULL, NULL); elapsed_balance += sc_MPI_Wtime (); /* postprocessing */ P4EST_GLOBAL_PRODUCTIONF ("Timings %g %g %g\n", elapsed_create, elapsed_partition, elapsed_balance); #ifdef BRICKS_VTK snprintf (filename, BUFSIZ, "brick_%02d_%02d_B", rlevel, l); p4est_vtk_write_file (p4est, NULL, filename); #endif p4est_destroy (p4est); p4est_connectivity_destroy (conn); }