int
p6est_vtk_write_header (p6est_t * p6est,
double scale, int write_tree, int write_rank,
int wrap_rank, const char *point_scalars,
const char *point_vectors, const char *filename)
{
p6est_connectivity_t *connectivity = p6est->connectivity;
p4est_t *p4est = p6est->columns;
sc_array_t *layers = p6est->layers;
sc_array_t *trees = p4est->trees;
const int mpirank = p4est->mpirank;
const double intsize = 1.0 / P4EST_ROOT_LEN;
double v[24];
const p4est_topidx_t first_local_tree = p4est->first_local_tree;
const p4est_topidx_t last_local_tree = p4est->last_local_tree;
const p4est_locidx_t Ncells = (p4est_locidx_t) layers->elem_count;
const p4est_locidx_t Ncorners = P8EST_CHILDREN * Ncells;
#ifdef P4EST_VTK_ASCII
double wx, wy, wz;
p4est_locidx_t sk;
#else
int retval;
uint8_t *uint8_data;
p4est_locidx_t *locidx_data;
#endif
int xi, yi, j, k;
int zi;
double h2, h2z, eta_x, eta_y, eta_z = 0.;
double xyz[3]; /* 3 not P4EST_DIM */
size_t num_cols, zz, zy, first, last;
p4est_topidx_t jt;
p4est_locidx_t quad_count, Ntotal;
p4est_locidx_t il;
P4EST_VTK_FLOAT_TYPE *float_data;
sc_array_t *columns;
p4est_tree_t *tree;
p4est_quadrant_t *col;
p2est_quadrant_t *layer;
char vtufilename[BUFSIZ];
FILE *vtufile;
SC_CHECK_ABORT (connectivity->conn4->num_vertices > 0,
"Must provide connectivity with vertex information");
P4EST_ASSERT (0. <= scale && scale <= 1. && wrap_rank >= 0);
Ntotal = Ncorners;
if (scale == 1.) {
scale = 1. - 2. * SC_EPS;
P4EST_ASSERT (scale < 1.);
}
/* Have each proc write to its own file */
snprintf (vtufilename, BUFSIZ, "%s_%04d.vtu", filename, mpirank);
/* Use "w" for writing the initial part of the file.
* For further parts, use "r+" and fseek so write_compressed succeeds.
*/
vtufile = fopen (vtufilename, "wb");
if (vtufile == NULL) {
P4EST_LERRORF ("Could not open %s for output\n", vtufilename);
return -1;
}
fprintf (vtufile, "<?xml version=\"1.0\"?>\n");
fprintf (vtufile, "<VTKFile type=\"UnstructuredGrid\" version=\"0.1\"");
#if defined P4EST_VTK_BINARY && defined P4EST_VTK_COMPRESSION
fprintf (vtufile, " compressor=\"vtkZLibDataCompressor\"");
#endif
#ifdef SC_IS_BIGENDIAN
fprintf (vtufile, " byte_order=\"BigEndian\">\n");
#else
fprintf (vtufile, " byte_order=\"LittleEndian\">\n");
#endif
fprintf (vtufile, " <UnstructuredGrid>\n");
fprintf (vtufile,
" <Piece NumberOfPoints=\"%lld\" NumberOfCells=\"%lld\">\n",
(long long) Ntotal, (long long) Ncells);
fprintf (vtufile, " <Points>\n");
float_data = P4EST_ALLOC (P4EST_VTK_FLOAT_TYPE, 3 * Ntotal);
/* write point position data */
fprintf (vtufile, " <DataArray type=\"%s\" Name=\"Position\""
" NumberOfComponents=\"3\" format=\"%s\">\n",
P4EST_VTK_FLOAT_NAME, P4EST_VTK_FORMAT_STRING);
/* loop over the trees */
for (jt = first_local_tree, quad_count = 0; jt <= last_local_tree; ++jt) {
tree = p4est_tree_array_index (trees, jt);
columns = &tree->quadrants;
num_cols = columns->elem_count;
p6est_tree_get_vertices (connectivity, jt, v);
/* loop over the elements in tree and calculated vertex coordinates */
for (zz = 0; zz < num_cols; ++zz) {
col = p4est_quadrant_array_index (columns, zz);
P6EST_COLUMN_GET_RANGE (col, &first, &last);
for (zy = first; zy < last; zy++, quad_count++) {
layer = p2est_quadrant_array_index (layers, zy);
h2 = .5 * intsize * P4EST_QUADRANT_LEN (col->level);
h2z = .5 * intsize * P4EST_QUADRANT_LEN (layer->level);
k = 0;
for (zi = 0; zi < 2; ++zi) {
for (yi = 0; yi < 2; ++yi) {
for (xi = 0; xi < 2; ++xi) {
P4EST_ASSERT (0 <= k && k < P8EST_CHILDREN);
eta_x = intsize * col->x + h2 * (1. + (xi * 2 - 1) * scale);
eta_y = intsize * col->y + h2 * (1. + (yi * 2 - 1) * scale);
eta_z = intsize * layer->z + h2z * (1. + (zi * 2 - 1) * scale);
for (j = 0; j < 3; ++j) {
/* *INDENT-OFF* */
xyz[j] =
((1. - eta_z) * ((1. - eta_y) * ((1. - eta_x) * v[3 * 0 + j] +
eta_x * v[3 * 1 + j]) +
eta_y * ((1. - eta_x) * v[3 * 2 + j] +
eta_x * v[3 * 3 + j]))
+ eta_z * ((1. - eta_y) * ((1. - eta_x) * v[3 * 4 + j] +
eta_x * v[3 * 5 + j]) +
eta_y * ((1. - eta_x) * v[3 * 6 + j] +
eta_x * v[3 * 7 + j]))
);
/* *INDENT-ON* */
}
for (j = 0; j < 3; ++j) {
float_data[3 * (P8EST_CHILDREN * quad_count + k) +
j] = (P4EST_VTK_FLOAT_TYPE) xyz[j];
}
++k;
}
}
}
P4EST_ASSERT (k == P8EST_CHILDREN);
}
}
}
P4EST_ASSERT (P8EST_CHILDREN * quad_count == Ntotal);
#ifdef P4EST_VTK_ASCII
for (il = 0; il < Ntotal; ++il) {
wx = float_data[3 * il + 0];
wy = float_data[3 * il + 1];
wz = float_data[3 * il + 2];
#ifdef P4EST_VTK_DOUBLES
fprintf (vtufile, " %24.16e %24.16e %24.16e\n", wx, wy, wz);
#else
fprintf (vtufile, " %16.8e %16.8e %16.8e\n", wx, wy, wz);
#endif
}
#else
fprintf (vtufile, " ");
/* TODO: Don't allocate the full size of the array, only allocate
* the chunk that will be passed to zlib and do this a chunk
* at a time.
*/
retval = p6est_vtk_write_binary (vtufile, (char *) float_data,
sizeof (*float_data) * 3 * Ntotal);
fprintf (vtufile, "\n");
if (retval) {
P4EST_LERROR ("p6est_vtk: Error encoding points\n");
fclose (vtufile);
return -1;
}
#endif
P4EST_FREE (float_data);
fprintf (vtufile, " </DataArray>\n");
fprintf (vtufile, " </Points>\n");
fprintf (vtufile, " <Cells>\n");
/* write connectivity data */
fprintf (vtufile, " <DataArray type=\"%s\" Name=\"connectivity\""
" format=\"%s\">\n", P4EST_VTK_LOCIDX, P4EST_VTK_FORMAT_STRING);
#ifdef P4EST_VTK_ASCII
for (sk = 0, il = 0; il < Ncells; ++il) {
fprintf (vtufile, " ");
for (k = 0; k < P8EST_CHILDREN; ++sk, ++k) {
fprintf (vtufile, " %lld", (long long) sk);
}
fprintf (vtufile, "\n");
}
#else
locidx_data = P4EST_ALLOC (p4est_locidx_t, Ncorners);
fprintf (vtufile, " ");
for (il = 0; il < Ncorners; ++il) {
locidx_data[il] = il;
}
retval = p6est_vtk_write_binary (vtufile, (char *) locidx_data,
sizeof (*locidx_data) * Ncorners);
fprintf (vtufile, "\n");
if (retval) {
P4EST_LERROR ("p6est_vtk: Error encoding connectivity\n");
fclose (vtufile);
return -1;
}
#endif
fprintf (vtufile, " </DataArray>\n");
/* write offset data */
fprintf (vtufile, " <DataArray type=\"%s\" Name=\"offsets\""
" format=\"%s\">\n", P4EST_VTK_LOCIDX, P4EST_VTK_FORMAT_STRING);
#ifdef P4EST_VTK_ASCII
fprintf (vtufile, " ");
for (il = 1, sk = 1; il <= Ncells; ++il, ++sk) {
fprintf (vtufile, " %lld", (long long) (P8EST_CHILDREN * il));
if (!(sk % 8) && il != Ncells)
fprintf (vtufile, "\n ");
}
fprintf (vtufile, "\n");
#else
for (il = 1; il <= Ncells; ++il)
locidx_data[il - 1] = P8EST_CHILDREN * il; /* same type */
fprintf (vtufile, " ");
retval = p6est_vtk_write_binary (vtufile, (char *) locidx_data,
sizeof (*locidx_data) * Ncells);
fprintf (vtufile, "\n");
if (retval) {
P4EST_LERROR ("p6est_vtk: Error encoding offsets\n");
fclose (vtufile);
return -1;
}
#endif
fprintf (vtufile, " </DataArray>\n");
/* write type data */
fprintf (vtufile, " <DataArray type=\"UInt8\" Name=\"types\""
" format=\"%s\">\n", P4EST_VTK_FORMAT_STRING);
#ifdef P4EST_VTK_ASCII
fprintf (vtufile, " ");
for (il = 0, sk = 1; il < Ncells; ++il, ++sk) {
fprintf (vtufile, " %d", P4EST_VTK_CELL_TYPE);
if (!(sk % 20) && il != (Ncells - 1))
fprintf (vtufile, "\n ");
}
fprintf (vtufile, "\n");
#else
uint8_data = P4EST_ALLOC (uint8_t, Ncells);
for (il = 0; il < Ncells; ++il)
uint8_data[il] = P4EST_VTK_CELL_TYPE;
fprintf (vtufile, " ");
retval = p6est_vtk_write_binary (vtufile, (char *) uint8_data,
sizeof (*uint8_data) * Ncells);
P4EST_FREE (uint8_data);
fprintf (vtufile, "\n");
if (retval) {
P4EST_LERROR ("p6est_vtk: Error encoding types\n");
fclose (vtufile);
return -1;
}
#endif
fprintf (vtufile, " </DataArray>\n");
fprintf (vtufile, " </Cells>\n");
if (write_rank || write_tree) {
fprintf (vtufile, " <CellData Scalars=\"%s\">\n",
!write_tree ? "mpirank" : !write_rank ? "treeid" :
"mpirank,treeid");
}
if (write_rank) {
const int wrapped_rank =
wrap_rank > 0 ? mpirank % wrap_rank : mpirank;
fprintf (vtufile, " <DataArray type=\"%s\" Name=\"mpirank\""
" format=\"%s\">\n", P4EST_VTK_LOCIDX, P4EST_VTK_FORMAT_STRING);
#ifdef P4EST_VTK_ASCII
fprintf (vtufile, " ");
for (il = 0, sk = 1; il < Ncells; ++il, ++sk) {
fprintf (vtufile, " %d", wrapped_rank);
if (!(sk % 20) && il != (Ncells - 1))
fprintf (vtufile, "\n ");
}
fprintf (vtufile, "\n");
#else
for (il = 0; il < Ncells; ++il)
locidx_data[il] = (p4est_locidx_t) wrapped_rank;
fprintf (vtufile, " ");
retval = p6est_vtk_write_binary (vtufile, (char *) locidx_data,
sizeof (*locidx_data) * Ncells);
fprintf (vtufile, "\n");
if (retval) {
P4EST_LERROR ("p6est_vtk: Error encoding types\n");
fclose (vtufile);
return -1;
}
#endif
fprintf (vtufile, " </DataArray>\n");
}
if (write_tree) {
fprintf (vtufile, " <DataArray type=\"%s\" Name=\"treeid\""
" format=\"%s\">\n", P4EST_VTK_LOCIDX, P4EST_VTK_FORMAT_STRING);
#ifdef P4EST_VTK_ASCII
fprintf (vtufile, " ");
for (il = 0, sk = 1, jt = first_local_tree; jt <= last_local_tree; ++jt) {
tree = p4est_tree_array_index (trees, jt);
num_cols = tree->quadrants.elem_count;
columns = &tree->quadrants;
for (zz = 0; zz < num_cols; ++zz) {
col = p4est_quadrant_array_index (columns, zz);
P6EST_COLUMN_GET_RANGE (col, &first, &last);
for (zy = first; zy < last; zy++, sk++, il++) {
fprintf (vtufile, " %lld", (long long) jt);
if (!(sk % 20) && il != (Ncells - 1))
fprintf (vtufile, "\n ");
}
}
}
fprintf (vtufile, "\n");
#else
for (il = 0, jt = first_local_tree; jt <= last_local_tree; ++jt) {
tree = p4est_tree_array_index (trees, jt);
num_cols = tree->quadrants.elem_count;
columns = &tree->quadrants;
for (zz = 0; zz < num_cols; ++zz) {
col = p4est_quadrant_array_index (columns, zz);
P6EST_COLUMN_GET_RANGE (col, &first, &last);
for (zy = first; zy < last; zy++, il++) {
locidx_data[il] = (p4est_locidx_t) jt;
}
}
}
fprintf (vtufile, " ");
retval = p6est_vtk_write_binary (vtufile, (char *) locidx_data,
sizeof (*locidx_data) * Ncells);
fprintf (vtufile, "\n");
if (retval) {
P4EST_LERROR ("p6est_vtk: Error encoding types\n");
fclose (vtufile);
return -1;
}
#endif
fprintf (vtufile, " </DataArray>\n");
P4EST_ASSERT (il == Ncells);
}
if (write_rank || write_tree) {
fprintf (vtufile, " </CellData>\n");
}
#ifndef P4EST_VTK_ASCII
P4EST_FREE (locidx_data);
#endif
fprintf (vtufile, " <PointData");
if (point_scalars != NULL)
fprintf (vtufile, " Scalars=\"%s\"", point_scalars);
if (point_vectors != NULL)
fprintf (vtufile, " Vectors=\"%s\"", point_vectors);
fprintf (vtufile, ">\n");
if (ferror (vtufile)) {
P4EST_LERROR ("p6est_vtk: Error writing header\n");
fclose (vtufile);
return -1;
}
if (fclose (vtufile)) {
P4EST_LERROR ("p6est_vtk: Error closing header\n");
return -1;
}
vtufile = NULL;
/* Only have the root write to the parallel vtk file */
if (mpirank == 0) {
char pvtufilename[BUFSIZ];
FILE *pvtufile;
snprintf (pvtufilename, BUFSIZ, "%s.pvtu", filename);
pvtufile = fopen (pvtufilename, "wb");
if (!pvtufile) {
P4EST_LERRORF ("Could not open %s for output\n", vtufilename);
return -1;
}
fprintf (pvtufile, "<?xml version=\"1.0\"?>\n");
fprintf (pvtufile, "<VTKFile type=\"PUnstructuredGrid\" version=\"0.1\"");
#if defined P4EST_VTK_BINARY && defined P4EST_VTK_COMPRESSION
fprintf (pvtufile, " compressor=\"vtkZLibDataCompressor\"");
#endif
#ifdef SC_IS_BIGENDIAN
fprintf (pvtufile, " byte_order=\"BigEndian\">\n");
#else
fprintf (pvtufile, " byte_order=\"LittleEndian\">\n");
#endif
fprintf (pvtufile, " <PUnstructuredGrid GhostLevel=\"0\">\n");
fprintf (pvtufile, " <PPoints>\n");
fprintf (pvtufile, " <PDataArray type=\"%s\" Name=\"Position\""
" NumberOfComponents=\"3\" format=\"%s\"/>\n",
P4EST_VTK_FLOAT_NAME, P4EST_VTK_FORMAT_STRING);
fprintf (pvtufile, " </PPoints>\n");
if (write_rank || write_tree) {
fprintf (pvtufile, " <PCellData Scalars=\"%s\">\n",
!write_tree ? "mpirank" : !write_rank ? "treeid" :
"mpirank,treeid");
}
if (write_rank) {
fprintf (pvtufile, " "
"<PDataArray type=\"%s\" Name=\"mpirank\" format=\"%s\"/>\n",
P4EST_VTK_LOCIDX, P4EST_VTK_FORMAT_STRING);
}
if (write_tree) {
fprintf (pvtufile, " "
"<PDataArray type=\"%s\" Name=\"treeid\" format=\"%s\"/>\n",
P4EST_VTK_LOCIDX, P4EST_VTK_FORMAT_STRING);
}
if (write_rank || write_tree) {
fprintf (pvtufile, " </PCellData>\n");
}
fprintf (pvtufile, " <PPointData>\n");
if (ferror (pvtufile)) {
P4EST_LERROR ("p6est_vtk: Error writing parallel header\n");
fclose (pvtufile);
return -1;
}
if (fclose (pvtufile)) {
P4EST_LERROR ("p6est_vtk: Error closing parallel header\n");
return -1;
}
}
return 0;
}
static void
mesh_run (mpi_context_t * mpi, p4est_connectivity_t * connectivity,
int uniform, int compute_tree_index, int compute_level_lists,
p4est_connect_type_t mesh_btype)
{
int mpiret;
unsigned crc;
long local_used[4], global_used[4];
p4est_t *p4est;
p4est_ghost_t *ghost;
p4est_mesh_t *mesh;
user_data_t *ghost_data;
p4est = p4est_new (mpi->mpicomm, connectivity,
sizeof (user_data_t), init_fn, NULL);
if (!uniform)
p4est_vtk_write_file (p4est, NULL, P4EST_STRING "_mesh_new");
/* refinement */
if (uniform) {
p4est_refine (p4est, 1, refine_uniform, init_fn);
}
else {
p4est_refine (p4est, 1, refine_normal, init_fn);
p4est_vtk_write_file (p4est, NULL, P4EST_STRING "_mesh_refined");
}
/* balance */
p4est_balance (p4est, P4EST_CONNECT_FULL, init_fn);
if (!uniform)
p4est_vtk_write_file (p4est, NULL, P4EST_STRING "_mesh_balanced");
/* partition */
p4est_partition (p4est, 0, NULL);
if (!uniform) {
p4est_vtk_write_file (p4est, NULL, P4EST_STRING "_mesh_partition");
}
crc = p4est_checksum (p4est);
/* print and verify forest checksum */
P4EST_GLOBAL_STATISTICSF ("Tree %s checksum 0x%08x\n",
uniform ? "uniform" : "adapted", crc);
/* create ghost layer and mesh */
ghost = p4est_ghost_new (p4est, P4EST_CONNECT_FULL);
ghost_data = P4EST_ALLOC (user_data_t, ghost->ghosts.elem_count);
p4est_ghost_exchange_data (p4est, ghost, ghost_data);
mesh = p4est_mesh_new_ext (p4est, ghost,
compute_tree_index, compute_level_lists,
mesh_btype);
test_mesh (p4est, ghost, mesh,
compute_tree_index, compute_level_lists, mesh_btype,
ghost_data, uniform);
/* compute memory used */
local_used[0] = (long) p4est_connectivity_memory_used (p4est->connectivity);
local_used[1] = (long) p4est_memory_used (p4est);
local_used[2] = (long) p4est_ghost_memory_used (ghost);
local_used[3] = (long) p4est_mesh_memory_used (mesh);
mpiret = sc_MPI_Allreduce (local_used, global_used, 4, sc_MPI_LONG,
sc_MPI_SUM, mpi->mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_PRODUCTIONF ("Total %s memory used %ld %ld %ld %ld\n",
uniform ? "uniform" : "adapted",
global_used[0], global_used[1],
global_used[2], global_used[3]);
/* destroy ghost layer and mesh */
P4EST_FREE (ghost_data);
p4est_mesh_destroy (mesh);
p4est_ghost_destroy (ghost);
/* destroy the p4est structure */
p4est_destroy (p4est);
}
static void
p4est_check_local_order (p4est_t * p4est, p4est_connectivity_t * connectivity)
{
const double intsize = 1.0 / P4EST_ROOT_LEN;
double *vertices;
double h, eta1, eta2;
double v0x, v0y, v0z, v1x, v1y, v1z;
double v2x, v2y, v2z, v3x, v3y, v3z;
double w0x, w0y, w0z, w1x, w1y, w1z;
double w2x, w2y, w2z, w3x, w3y, w3z;
size_t iz;
size_t num_quads;
size_t quad_count;
p4est_topidx_t jt;
p4est_topidx_t *tree_to_vertex;
p4est_topidx_t first_local_tree;
p4est_topidx_t last_local_tree;
p4est_topidx_t v0, v1, v2, v3;
p4est_locidx_t kl;
p4est_locidx_t lv0, lv1, lv2, lv3;
p4est_locidx_t num_uniq_local_vertices;
p4est_locidx_t *quadrant_to_local_vertex;
p4est_qcoord_t inth;
p4est_tree_t *tree;
p4est_quadrant_t *quad;
p4est_vert_t *vert_locations;
p4est_nodes_t *nodes;
sc_array_t *trees;
sc_array_t *quadrants;
nodes = p4est_nodes_new (p4est, NULL);
quadrant_to_local_vertex = nodes->local_nodes;
num_uniq_local_vertices = nodes->num_owned_indeps;
SC_CHECK_ABORT ((size_t) num_uniq_local_vertices ==
nodes->indep_nodes.elem_count, "Node count mismatch");
P4EST_INFOF ("Unique local vertices %lld\n",
(long long) num_uniq_local_vertices);
vert_locations = P4EST_ALLOC (p4est_vert_t, num_uniq_local_vertices);
for (kl = 0; kl < num_uniq_local_vertices; ++kl) {
vert_locations[kl].treeid = -1;
}
tree_to_vertex = connectivity->tree_to_vertex;
vertices = connectivity->vertices;
first_local_tree = p4est->first_local_tree;
last_local_tree = p4est->last_local_tree;
trees = p4est->trees;
for (jt = first_local_tree, quad_count = 0; jt <= last_local_tree; ++jt) {
tree = p4est_tree_array_index (trees, jt);
P4EST_ASSERT (0 <= jt && jt < connectivity->num_trees);
v0 = tree_to_vertex[jt * 4 + 0];
v1 = tree_to_vertex[jt * 4 + 1];
v2 = tree_to_vertex[jt * 4 + 2];
v3 = tree_to_vertex[jt * 4 + 3];
P4EST_ASSERT (0 <= v0 && v0 < connectivity->num_vertices);
P4EST_ASSERT (0 <= v1 && v1 < connectivity->num_vertices);
P4EST_ASSERT (0 <= v2 && v2 < connectivity->num_vertices);
P4EST_ASSERT (0 <= v3 && v3 < connectivity->num_vertices);
v0x = vertices[v0 * 3 + 0];
v0y = vertices[v0 * 3 + 1];
v0z = vertices[v0 * 3 + 2];
v1x = vertices[v1 * 3 + 0];
v1y = vertices[v1 * 3 + 1];
v1z = vertices[v1 * 3 + 2];
v2x = vertices[v2 * 3 + 0];
v2y = vertices[v2 * 3 + 1];
v2z = vertices[v2 * 3 + 2];
v3x = vertices[v3 * 3 + 0];
v3y = vertices[v3 * 3 + 1];
v3z = vertices[v3 * 3 + 2];
quadrants = &tree->quadrants;
num_quads = quadrants->elem_count;
/* loop over the elements in the tree */
for (iz = 0; iz < num_quads; ++iz, ++quad_count) {
quad = p4est_quadrant_array_index (quadrants, iz);
inth = P4EST_QUADRANT_LEN (quad->level);
h = intsize * inth;
eta1 = intsize * quad->x;
eta2 = intsize * quad->y;
w0x = v0x * (1.0 - eta1) * (1.0 - eta2)
+ v1x * (eta1) * (1.0 - eta2)
+ v2x * (1.0 - eta1) * (eta2)
+ v3x * (eta1) * (eta2);
w0y = v0y * (1.0 - eta1) * (1.0 - eta2)
+ v1y * (eta1) * (1.0 - eta2)
+ v2y * (1.0 - eta1) * (eta2)
+ v3y * (eta1) * (eta2);
w0z = v0z * (1.0 - eta1) * (1.0 - eta2)
+ v1z * (eta1) * (1.0 - eta2)
+ v2z * (1.0 - eta1) * (eta2)
+ v3z * (eta1) * (eta2);
w1x = v0x * (1.0 - eta1 - h) * (1.0 - eta2)
+ v1x * (eta1 + h) * (1.0 - eta2)
+ v2x * (1.0 - eta1 - h) * (eta2)
+ v3x * (eta1 + h) * (eta2);
w1y = v0y * (1.0 - eta1 - h) * (1.0 - eta2)
+ v1y * (eta1 + h) * (1.0 - eta2)
+ v2y * (1.0 - eta1 - h) * (eta2)
+ v3y * (eta1 + h) * (eta2);
w1z = v0z * (1.0 - eta1 - h) * (1.0 - eta2)
+ v1z * (eta1 + h) * (1.0 - eta2)
+ v2z * (1.0 - eta1 - h) * (eta2)
+ v3z * (eta1 + h) * (eta2);
w2x = v0x * (1.0 - eta1) * (1.0 - eta2 - h)
+ v1x * (eta1) * (1.0 - eta2 - h)
+ v2x * (1.0 - eta1) * (eta2 + h)
+ v3x * (eta1) * (eta2 + h);
w2y = v0y * (1.0 - eta1) * (1.0 - eta2 - h)
+ v1y * (eta1) * (1.0 - eta2 - h)
+ v2y * (1.0 - eta1) * (eta2 + h)
+ v3y * (eta1) * (eta2 + h);
w2z = v0z * (1.0 - eta1) * (1.0 - eta2 - h)
+ v1z * (eta1) * (1.0 - eta2 - h)
+ v2z * (1.0 - eta1) * (eta2 + h)
+ v3z * (eta1) * (eta2 + h);
w3x = v0x * (1.0 - eta1 - h) * (1.0 - eta2 - h)
+ v1x * (eta1 + h) * (1.0 - eta2 - h)
+ v2x * (1.0 - eta1 - h) * (eta2 + h)
+ v3x * (eta1 + h) * (eta2 + h);
w3y = v0y * (1.0 - eta1 - h) * (1.0 - eta2 - h)
+ v1y * (eta1 + h) * (1.0 - eta2 - h)
+ v2y * (1.0 - eta1 - h) * (eta2 + h)
+ v3y * (eta1 + h) * (eta2 + h);
w3z = v0z * (1.0 - eta1 - h) * (1.0 - eta2 - h)
+ v1z * (eta1 + h) * (1.0 - eta2 - h)
+ v2z * (1.0 - eta1 - h) * (eta2 + h)
+ v3z * (eta1 + h) * (eta2 + h);
P4EST_ASSERT ((p4est_locidx_t) quad_count < p4est->local_num_quadrants);
lv0 = quadrant_to_local_vertex[4 * quad_count + 0];
lv1 = quadrant_to_local_vertex[4 * quad_count + 1];
lv2 = quadrant_to_local_vertex[4 * quad_count + 2];
lv3 = quadrant_to_local_vertex[4 * quad_count + 3];
P4EST_ASSERT (0 <= lv0 && lv0 < num_uniq_local_vertices);
P4EST_ASSERT (0 <= lv1 && lv1 < num_uniq_local_vertices);
P4EST_ASSERT (0 <= lv2 && lv2 < num_uniq_local_vertices);
P4EST_ASSERT (0 <= lv3 && lv3 < num_uniq_local_vertices);
vert_locations[lv0].x = w0x;
vert_locations[lv0].y = w0y;
vert_locations[lv0].z = w0z;
P4EST_ASSERT (vert_locations[lv0].treeid == -1 ||
vert_locations[lv0].treeid == jt);
vert_locations[lv0].treeid = jt;
vert_locations[lv1].x = w1x;
vert_locations[lv1].y = w1y;
vert_locations[lv1].z = w1z;
P4EST_ASSERT (vert_locations[lv1].treeid == -1 ||
vert_locations[lv1].treeid == jt);
vert_locations[lv1].treeid = jt;
vert_locations[lv2].x = w2x;
vert_locations[lv2].y = w2y;
vert_locations[lv2].z = w2z;
P4EST_ASSERT (vert_locations[lv2].treeid == -1 ||
vert_locations[lv2].treeid == jt);
vert_locations[lv2].treeid = jt;
vert_locations[lv3].x = w3x;
vert_locations[lv3].y = w3y;
vert_locations[lv3].z = w3z;
P4EST_ASSERT (vert_locations[lv3].treeid == -1 ||
vert_locations[lv3].treeid == jt);
vert_locations[lv3].treeid = jt;
}
}
qsort (vert_locations, num_uniq_local_vertices, sizeof (p4est_vert_t),
p4est_vert_compare);
/* Check to make sure that we don't have any duplicates in the list */
for (kl = 0; kl < num_uniq_local_vertices - 1; ++kl) {
SC_CHECK_ABORT (p4est_vert_compare (vert_locations + kl,
vert_locations + kl + 1) != 0,
"local ordering not unique");
}
P4EST_FREE (vert_locations);
p4est_nodes_destroy (nodes);
}
p6est_profile_t *
p6est_profile_new_local (p6est_t * p6est,
p6est_ghost_t * ghost,
p6est_profile_type_t ptype,
p8est_connect_type_t btype, int degree)
{
p6est_profile_t *profile = P4EST_ALLOC (p6est_profile_t, 1);
p4est_lnodes_t *lnodes;
p4est_locidx_t nln, nle;
p4est_topidx_t jt;
p4est_t *columns = p6est->columns;
p4est_tree_t *tree;
sc_array_t *tquadrants;
p4est_quadrant_t *col;
p4est_qcoord_t diff = P4EST_ROOT_LEN - p6est->root_len;
size_t first, last, count, zz, zy;
p4est_locidx_t *en, (*lr)[2];
sc_array_t *lc;
int i, j;
p2est_quadrant_t *layer;
sc_array_t *layers = p6est->layers;
p4est_locidx_t nidx, enidx;
p4est_connect_type_t hbtype;
int8_t *c;
sc_array_t *thisprof;
sc_array_t *selfprof;
sc_array_t *faceprof;
sc_array_t *cornerprof;
sc_array_t *work;
sc_array_t oldprof;
const int Nrp = degree + 1;
P4EST_ASSERT (degree > 1);
profile->ptype = ptype;
profile->btype = btype;
profile->lnode_changed[0] = NULL;
profile->lnode_changed[1] = NULL;
profile->enode_counts = NULL;
profile->diff = diff;
if (btype == P8EST_CONNECT_FACE) {
hbtype = P4EST_CONNECT_FACE;
}
else {
hbtype = P4EST_CONNECT_FULL;
}
if (ghost == NULL) {
profile->cghost = p4est_ghost_new (p6est->columns, P4EST_CONNECT_FULL);
profile->ghost_owned = 1;
}
else {
P4EST_ASSERT (ghost->column_ghost->btype == P4EST_CONNECT_FULL);
profile->cghost = ghost->column_ghost;
profile->ghost_owned = 0;
}
if (ptype == P6EST_PROFILE_UNION) {
P4EST_ASSERT (degree == 2);
}
profile->lnodes = lnodes = p4est_lnodes_new (p6est->columns,
profile->cghost, degree);
en = lnodes->element_nodes;
nln = lnodes->num_local_nodes;
nle = lnodes->num_local_elements;
profile->lnode_ranges = P4EST_ALLOC_ZERO (p4est_locidx_t, 2 * nln);
lr = (p4est_locidx_t (*)[2]) profile->lnode_ranges;
profile->lnode_columns = lc = sc_array_new (sizeof (int8_t));
selfprof = sc_array_new (sizeof (int8_t));
work = sc_array_new (sizeof (int8_t));
faceprof = sc_array_new (sizeof (int8_t));
cornerprof = sc_array_new (sizeof (int8_t));
if (ptype == P6EST_PROFILE_UNION) {
profile->lnode_changed[0] = P4EST_ALLOC (p4est_locidx_t, nln);
profile->lnode_changed[1] = P4EST_ALLOC (p4est_locidx_t, nln);
profile->enode_counts = P4EST_ALLOC (p4est_locidx_t, P4EST_INSUL * nle);
profile->evenodd = 0;
memset (profile->lnode_changed[0], -1, nln * sizeof (int));
}
/* create the profiles for each node: layers are reduced to just their level
* */
for (enidx = 0, jt = columns->first_local_tree;
jt <= columns->last_local_tree; ++jt) {
tree = p4est_tree_array_index (columns->trees, jt);
tquadrants = &tree->quadrants;
for (zz = 0; zz < tquadrants->elem_count; ++zz) {
col = p4est_quadrant_array_index (tquadrants, zz);
P6EST_COLUMN_GET_RANGE (col, &first, &last);
count = last - first;
sc_array_truncate (selfprof);
c = (int8_t *) sc_array_push_count (selfprof, count);
for (zy = first; zy < last; zy++) {
layer = p2est_quadrant_array_index (layers, zy);
*(c++) = layer->level;
}
if (ptype == P6EST_PROFILE_UNION) {
p6est_profile_balance_self (selfprof, work);
if (btype == P8EST_CONNECT_FACE) {
p6est_profile_balance_face (selfprof, faceprof, work, diff);
}
else {
p6est_profile_balance_full (selfprof, faceprof, work, diff);
}
if (btype == P8EST_CONNECT_EDGE) {
p6est_profile_balance_face (selfprof, cornerprof, work, diff);
}
else if (btype == P8EST_CONNECT_FULL) {
p6est_profile_balance_full (selfprof, cornerprof, work, diff);
}
}
for (j = 0; j < Nrp; j++) {
for (i = 0; i < Nrp; i++, enidx++) {
nidx = en[enidx];
if (ptype == P6EST_PROFILE_UNION) {
thisprof = NULL;
if (!(i % degree) && !(j % degree)) {
if (hbtype == P4EST_CONNECT_FACE) {
/* skip corners if we don't need to balance them */
P4EST_ASSERT (!lr[nidx][0]);
P4EST_ASSERT (!lr[nidx][1]);
continue;
}
else {
thisprof = cornerprof;
}
}
else if ((i % degree) && (j % degree)) {
thisprof = selfprof;
}
else {
thisprof = faceprof;
}
count = thisprof->elem_count;
profile->enode_counts[enidx] = count;
if (!lr[nidx][1]) {
/* if this node has not yet been initialized, initialize it */
lr[nidx][0] = lc->elem_count;
lr[nidx][1] = count;
c = (int8_t *) sc_array_push_count (lc, count);
memcpy (c, thisprof->array, count * sizeof (int8_t));
}
else {
/* if this node has been initialized, combine the two profiles,
* taking the finer layers from each */
sc_array_init_view (&oldprof, lc, lr[nidx][0], lr[nidx][1]);
p6est_profile_union (thisprof, &oldprof, work);
if (work->elem_count > oldprof.elem_count) {
lr[nidx][0] = lc->elem_count;
lr[nidx][1] = work->elem_count;
c = (int8_t *) sc_array_push_count (lc, work->elem_count);
memcpy (c, work->array, work->elem_count * work->elem_size);
}
}
}
else {
count = selfprof->elem_count;
if (!lr[nidx][1]) {
/* if this node has not yet been initialized, initialize it */
lr[nidx][0] = lc->elem_count;
lr[nidx][1] = count;
c = (int8_t *) sc_array_push_count (lc, count);
memcpy (c, selfprof->array, count * sizeof (int8_t));
}
else {
/* if this node has been initialized, combine the two profiles,
* taking the coarser layers from each */
sc_array_init_view (&oldprof, lc, lr[nidx][0], lr[nidx][1]);
p6est_profile_intersection (selfprof, &oldprof, work);
P4EST_ASSERT (work->elem_count <= oldprof.elem_count);
if (work->elem_count < oldprof.elem_count) {
lr[nidx][1] = work->elem_count;
memcpy (oldprof.array, work->array,
work->elem_count * work->elem_size);
}
}
}
}
}
}
}
p6est_profile_compress (profile);
sc_array_destroy (selfprof);
sc_array_destroy (faceprof);
sc_array_destroy (cornerprof);
sc_array_destroy (work);
return profile;
}
int
p6est_profile_sync (p6est_profile_t * profile)
{
p4est_lnodes_t *lnodes = profile->lnodes;
p4est_locidx_t nln = lnodes->num_local_nodes;
sc_array_t lrview;
p4est_lnodes_buffer_t *countbuf;
sc_array_t *sharers;
size_t zz, nsharers;
int nleft;
int8_t *recv, *send;
int *array_of_indices;
p4est_locidx_t recv_total;
p4est_locidx_t *recv_offsets, recv_offset;
p4est_locidx_t send_total;
p4est_locidx_t *send_offsets, send_offset;
p4est_locidx_t (*lr)[2];
sc_array_t *lc = profile->lnode_columns;
sc_MPI_Request *recv_request, *send_request;
sc_array_t *work;
int any_change = 0;
int any_global_change;
int mpiret, mpirank;
int evenodd = profile->evenodd;
lr = (p4est_locidx_t (*)[2]) profile->lnode_ranges;
sharers = lnodes->sharers;
nsharers = sharers->elem_count;
mpiret = sc_MPI_Comm_rank (lnodes->mpicomm, &mpirank);
SC_CHECK_MPI (mpiret);
sc_array_init_data (&lrview, lr, 2 * sizeof (p4est_locidx_t), nln);
countbuf = p4est_lnodes_share_all_begin (&lrview, lnodes);
send_offsets = P4EST_ALLOC (p4est_locidx_t, nsharers + 1);
send_offset = 0;
for (zz = 0; zz < nsharers; zz++) {
p4est_lnodes_rank_t *sharer;
sc_array_t *send_buf;
size_t zy, nnodes;
send_offsets[zz] = send_offset;
sharer = p4est_lnodes_rank_array_index (sharers, zz);
if (sharer->rank == mpirank) {
continue;
}
send_buf = (sc_array_t *) sc_array_index (countbuf->send_buffers, zz);
nnodes = sharer->shared_nodes.elem_count;
P4EST_ASSERT (nnodes == send_buf->elem_count);
P4EST_ASSERT (send_buf->elem_size == 2 * sizeof (p4est_locidx_t));
for (zy = 0; zy < nnodes; zy++) {
p4est_locidx_t *lp =
(p4est_locidx_t *) sc_array_index (send_buf, zy);
P4EST_ASSERT (lp[0] >= 0);
P4EST_ASSERT (lp[1] >= 0);
send_offset += lp[1];
}
}
send_total = send_offsets[nsharers] = send_offset;
p4est_lnodes_share_all_end (countbuf);
recv_offsets = P4EST_ALLOC (p4est_locidx_t, nsharers + 1);
recv_offset = 0;
for (zz = 0; zz < nsharers; zz++) {
p4est_lnodes_rank_t *sharer;
sc_array_t *recv_buf;
size_t zy, nnodes;
recv_offsets[zz] = recv_offset;
sharer = p4est_lnodes_rank_array_index (sharers, zz);
if (sharer->rank == mpirank) {
continue;
}
recv_buf = (sc_array_t *) sc_array_index (countbuf->recv_buffers, zz);
nnodes = sharer->shared_nodes.elem_count;
P4EST_ASSERT (nnodes == recv_buf->elem_count);
P4EST_ASSERT (recv_buf->elem_size == 2 * sizeof (p4est_locidx_t));
for (zy = 0; zy < nnodes; zy++) {
p4est_locidx_t *lp =
(p4est_locidx_t *) sc_array_index (recv_buf, zy);
P4EST_ASSERT (lp[0] >= 0);
P4EST_ASSERT (lp[1] >= 0);
recv_offset += lp[1];
}
}
recv_total = recv_offsets[nsharers] = recv_offset;
recv = P4EST_ALLOC (int8_t, recv_total);
recv_request = P4EST_ALLOC (sc_MPI_Request, nsharers);
send = P4EST_ALLOC (int8_t, send_total);
send_request = P4EST_ALLOC (sc_MPI_Request, nsharers);
/* post receives */
nleft = 0;
for (zz = 0; zz < nsharers; zz++) {
p4est_lnodes_rank_t *sharer;
int icount = recv_offsets[zz + 1] - recv_offsets[zz];
sharer = p4est_lnodes_rank_array_index (sharers, zz);
if (sharer->rank == mpirank) {
recv_request[zz] = sc_MPI_REQUEST_NULL;
continue;
}
if (icount) {
mpiret =
sc_MPI_Irecv (recv + recv_offsets[zz], icount * sizeof (int8_t),
sc_MPI_BYTE, sharer->rank, P6EST_COMM_BALANCE,
lnodes->mpicomm, recv_request + zz);
SC_CHECK_MPI (mpiret);
nleft++;
}
else {
recv_request[zz] = sc_MPI_REQUEST_NULL;
}
}
/* post sends */
for (zz = 0; zz < nsharers; zz++) {
p4est_lnodes_rank_t *sharer;
size_t zy, nnodes;
int icount;
sc_array_t *shared_nodes;
sharer = p4est_lnodes_rank_array_index (sharers, zz);
if (sharer->rank == mpirank) {
send_request[zz] = sc_MPI_REQUEST_NULL;
continue;
}
shared_nodes = &sharer->shared_nodes;
nnodes = shared_nodes->elem_count;
icount = 0;
for (zy = 0; zy < nnodes; zy++) {
p4est_locidx_t nidx;
int8_t *c;
nidx = *((p4est_locidx_t *) sc_array_index (shared_nodes, zy));
if (lr[nidx][1]) {
c = (int8_t *) sc_array_index (lc, lr[nidx][0]);
memcpy (send + send_offsets[zz] + icount, c,
lr[nidx][1] * sizeof (int8_t));
icount += lr[nidx][1];
}
else {
P4EST_ASSERT (!lr[nidx][0]);
}
}
P4EST_ASSERT (icount == send_offsets[zz + 1] - send_offsets[zz]);
if (icount) {
mpiret =
sc_MPI_Isend (send + send_offsets[zz], icount * sizeof (int8_t),
sc_MPI_BYTE, sharer->rank, P6EST_COMM_BALANCE,
lnodes->mpicomm, send_request + zz);
SC_CHECK_MPI (mpiret);
}
else {
send_request[zz] = sc_MPI_REQUEST_NULL;
}
}
work = sc_array_new (sizeof (int8_t));
array_of_indices = P4EST_ALLOC (int, nsharers);
while (nleft) {
int outcount;
int i;
mpiret = sc_MPI_Waitsome (nsharers, recv_request, &outcount,
array_of_indices, sc_MPI_STATUSES_IGNORE);
SC_CHECK_MPI (mpiret);
for (i = 0; i < outcount; i++) {
p4est_lnodes_rank_t *sharer;
size_t zy, nnode;
sc_array_t *shared_nodes;
sc_array_t *recv_buf;
zz = array_of_indices[i];
sharer = p4est_lnodes_rank_array_index (sharers, zz);
shared_nodes = &sharer->shared_nodes;
recv_buf = (sc_array_t *) sc_array_index (countbuf->recv_buffers, zz);
nnode = shared_nodes->elem_count;
P4EST_ASSERT (nnode == recv_buf->elem_count);
recv_offset = recv_offsets[zz];
for (zy = 0; zy < nnode; zy++) {
p4est_locidx_t *lp;
p4est_locidx_t nidx;
sc_array_t oldview, newview;
nidx = *((p4est_locidx_t *) sc_array_index (shared_nodes, zy));
lp = (p4est_locidx_t *) sc_array_index (recv_buf, zy);
sc_array_init_view (&oldview, lc, lr[nidx][0], lr[nidx][1]);
sc_array_init_data (&newview, recv + recv_offset, sizeof (int8_t),
lp[1]);
if (profile->ptype == P6EST_PROFILE_UNION) {
p6est_profile_union (&oldview, &newview, work);
if (work->elem_count > oldview.elem_count) {
int8_t *c;
any_change = 1;
lr[nidx][0] = lc->elem_count;
lr[nidx][1] = work->elem_count;
profile->lnode_changed[evenodd][nidx] = 1;
c = (int8_t *) sc_array_push_count (lc, work->elem_count);
memcpy (c, work->array, work->elem_count * work->elem_size);
}
}
else {
p6est_profile_intersection (&oldview, &newview, work);
P4EST_ASSERT (work->elem_count <= oldview.elem_count);
if (work->elem_count < oldview.elem_count) {
lr[nidx][1] = work->elem_count;
memcpy (oldview.array, work->array,
work->elem_count * work->elem_size);
}
}
recv_offset += lp[1];
}
P4EST_ASSERT (recv_offset == recv_offsets[zz + 1]);
}
nleft -= outcount;
P4EST_ASSERT (nleft >= 0);
}
P4EST_FREE (array_of_indices);
sc_array_destroy (work);
p6est_profile_compress (profile);
p4est_lnodes_buffer_destroy (countbuf);
P4EST_FREE (recv_request);
P4EST_FREE (recv_offsets);
P4EST_FREE (recv);
{
mpiret = sc_MPI_Waitall (nsharers, send_request, sc_MPI_STATUSES_IGNORE);
SC_CHECK_MPI (mpiret);
P4EST_FREE (send_request);
P4EST_FREE (send_offsets);
P4EST_FREE (send);
any_global_change = any_change;
mpiret = sc_MPI_Allreduce (&any_change, &any_global_change, 1, sc_MPI_INT,
sc_MPI_LOR, lnodes->mpicomm);
SC_CHECK_MPI (mpiret);
}
return any_global_change;
}
int main(int argc, char *argv[])
{
int mpiret;
sc_MPI_Comm mpicomm;
int proc_size;
p4est_connectivity_t *conn;
p4est_geometry_t *geom;
p4est_t* p4est;
int seed = time(NULL);
srand(seed);
/* MPI init */
mpiret = sc_MPI_Init(&argc, &argv);
SC_CHECK_MPI(mpiret);
mpicomm = sc_MPI_COMM_WORLD;
sc_init (mpicomm, 1, 1, NULL, SC_LP_ESSENTIAL);
mpiret = MPI_Comm_size(mpicomm, &proc_size);
SC_CHECK_MPI (mpiret);
/* pXest init */
p4est_init(NULL, SC_LP_PRODUCTION);
conn = p4est_connectivity_new_disk();
/* geom = p4est_geometry_new_connectivity(conn); */
geom = p4est_geometry_new_disk(conn,1.,2.);
p4est = p4est_new_ext (mpicomm, conn, -1, 0, 1,
sizeof(curved_element_data_t), NULL, NULL);
int world_rank,world_size;
sc_MPI_Comm_rank(sc_MPI_COMM_WORLD, &world_rank);
sc_MPI_Comm_size(sc_MPI_COMM_WORLD, &world_size);
/* start just-in-time dg-math */
dgmath_jit_dbase_t* dgmath_jit_dbase = dgmath_jit_dbase_init();
geometric_factors_t* geometric_factors = geometric_factors_init(p4est);
curved_element_data_init(p4est, geometric_factors, dgmath_jit_dbase, geom, 4);
/* int local_nodes = curved_element_data_get_local_nodes(p4est); */
/* double* u = P4EST_ALLOC(double, local_nodes); */
/* for (int i = 0; i < local_nodes; i++){ */
/* /\* u = x *\/ */
/* u[i] = geometric_factors->xyz[i]; */
/* } */
int num_of_refinements = 2;
/* p4est_vtk_write_all */
/* (p4est, */
/* geom, */
/* 0.99, */
/* 1, */
/* 1, */
/* 1, */
/* 0, */
/* 0, */
/* 0, */
/* "disk0" */
/* ); */
for (int i = 0; i < num_of_refinements; i++){
p4est_refine_ext (p4est, 0, -1, random_h_refine, NULL, refine_uniform_replace_callback);
p4est_balance_ext(p4est, P4EST_CONNECT_FACE, NULL, refine_uniform_replace_callback);
/* p4est_vtk_write_all */
/* (p4est, */
/* geom, */
/* 0.99, */
/* 1, */
/* 1, */
/* 1, */
/* 0, */
/* 0, */
/* 0, */
/* "disk" */
/* ); */
}
curved_element_data_init(p4est, geometric_factors, dgmath_jit_dbase, geom, -1);
int local_nodes = curved_element_data_get_local_nodes(p4est);
double* u = P4EST_ALLOC(double, local_nodes);
for (int i = 0; i < local_nodes; i++){
/* u = x */
u[i] = geometric_factors->xyz[i];
}
/* curved_element_data_init(p4est, geometric_factors, dgmath_jit_dbase, geom, -1); */
/* store vector */
curved_element_data_copy_from_vec_to_storage
(
p4est,
u
);
test_curved_data_t test_curved_data;
test_curved_data.mortar_err = 0.;
test_curved_data.hanging_proj_err = 0.;
test_curved_data.full_proj_err = 0.;
test_curved_data.print_data = 1;
test_curved_data.no_reorient = 0;
test_curved_data.geom = geom;
p4est_ghost_t* ghost = p4est_ghost_new (p4est, P4EST_CONNECT_FACE);
/* create space for storing the ghost data */
curved_element_data_t* ghost_data = P4EST_ALLOC (curved_element_data_t,
ghost->ghosts.elem_count);
p4est_ghost_exchange_data (p4est, ghost, ghost_data);
curved_compute_flux_user_data_t curved_compute_flux_user_data;
curved_compute_flux_user_data.dgmath_jit_dbase = dgmath_jit_dbase;
curved_flux_fcn_ptrs_t flux_fcns = (test_curved_data_fetch_fcns(&test_curved_data));
curved_compute_flux_user_data.flux_fcn_ptrs = &flux_fcns;
p4est->user_pointer = &curved_compute_flux_user_data;
p4est_iterate (p4est,
ghost,
(void *) ghost_data,
NULL,
curved_compute_flux_on_local_elements,
#if (P4EST_DIM)==3
NULL,
#endif
NULL);
test_curved_data.mortar_err = 0.;
if (world_rank == 0)
printf("mortar_err = %.20f\n",
test_curved_data.mortar_err
);
p4est_ghost_destroy (ghost);
P4EST_FREE (ghost_data);
ghost = NULL;
ghost_data = NULL;
/* curved_hp_amr(p4est, */
/* &u, */
/* test_nonconform_random_hp, */
/* NULL, */
/* NULL, */
/* NULL, */
/* dgmath_jit_dbase */
/* ); */
/* curved_element_data_init(p4est, geometric_factors, dgmath_jit_dbase, geom, -1); */
/* double* u_vertex = P4EST_ALLOC(double, p4est->local_num_quadrants*(P4EST_CHILDREN)); */
/* element_data_store_nodal_vec_in_vertex_array */
/* ( */
/* p4est, */
/* u, */
/* u_vertex */
/* ); */
/* char sol_save_as [500]; */
/* sprintf(sol_save_as, "%s_test_nonconform_sym_level_%d_u", P4EST_STRING, i); */
/* curved_hacked_p4est_vtk_write_all */
/* (p4est, */
/* NULL, */
/* 0.99, */
/* 0, */
/* 1, */
/* 1, */
/* 0, */
/* 1, */
/* 0, */
/* sol_save_as, */
/* "u", */
/* u_vertex */
/* ); */
/* P4EST_FREE(u_vertex); */
/* ip_flux_params_t ip_flux_params; */
/* ip_flux_params.ip_flux_penalty_prefactor = atoi(argv[6]); */
/* ip_flux_params.ip_flux_penalty_calculate_fcn = sipg_flux_vector_calc_penalty_maxp2_over_minh; */
/* problem_data_t vecs; */
/* vecs.u = u; */
/* vecs.local_nodes = element_data_get_local_nodes(p4est); */
/* vecs.vector_flux_fcn_data = sipg_flux_vector_dirichlet_fetch_fcns */
/* ( */
/* zero_fcn, */
/* &ip_flux_params */
/* ); */
/* vecs.scalar_flux_fcn_data = sipg_flux_scalar_dirichlet_fetch_fcns(zero_fcn); */
/* weakeqn_ptrs_t fcns; */
/* fcns.apply_lhs = poisson_apply_aij; */
/* matrix_sym_tester */
/* ( */
/* p4est, */
/* &vecs, /\* only needed for # of nodes *\/ */
/* &fcns, */
/* .000000000000001, */
/* dgmath_jit_dbase, */
/* 0, */
/* 0 */
/* ); */
/* } */
P4EST_FREE(u);
geometric_factors_destroy(geometric_factors);
/* free pointers */
dgmath_jit_dbase_destroy(dgmath_jit_dbase);
/* free pXest */
p4est_destroy(p4est);
/* p4est_destroy(p4est); */
if (geom != NULL) {
p4est_geometry_destroy (geom);
}
p4est_connectivity_destroy(conn);
/* finalize mpi stuff */
sc_finalize();
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI(mpiret);
}
static void
mesh_iter_corner (p4est_iter_corner_info_t * info, void *user_data)
{
int i, j;
int f1, f2, code, orientation;
int fc1, fc2, diagonal;
#ifdef P4_TO_P8
int pref, pset;
#endif
int visited[P4EST_CHILDREN];
int8_t *pccorner;
size_t cz, zz;
sc_array_t *trees;
p4est_locidx_t qoffset, qid1, qid2;
p4est_locidx_t cornerid_offset, cornerid;
p4est_locidx_t *pcquad;
p4est_mesh_t *mesh = (p4est_mesh_t *) user_data;
p4est_iter_corner_side_t *side1, *side2;
p4est_tree_t *tree1, *tree2;
p4est_connectivity_t *conn;
/* Check the case when the corner does not involve neighbors */
cz = info->sides.elem_count;
P4EST_ASSERT (cz > 0);
P4EST_ASSERT (info->tree_boundary || cz == P4EST_CHILDREN);
if (cz == 1) {
return;
}
conn = info->p4est->connectivity;
trees = info->p4est->trees;
cornerid_offset = mesh->local_num_quadrants + mesh->ghost_num_quadrants;
if (info->tree_boundary == P4EST_CONNECT_FACE) {
/* This corner is inside an inter-tree face */
if (cz == P4EST_HALF) {
/* This is a tree face boundary, no corner neighbors exist */
return;
}
P4EST_ASSERT (cz == P4EST_CHILDREN);
/* Process a corner in pairs of diagonal inter-tree neighbors */
memset (visited, 0, P4EST_CHILDREN * sizeof (int));
for (i = 0; i < P4EST_HALF; ++i) {
side1 = side2 = NULL;
f1 = -1;
fc1 = -1;
qid1 = -3;
for (j = 0; j < P4EST_CHILDREN; ++j) {
if (visited[j]) {
continue;
}
/* Remember the first side we want to pair up */
if (side1 == NULL) {
side1 =
(p4est_iter_corner_side_t *) sc_array_index_int (&info->sides, j);
P4EST_ASSERT (side1->quad != NULL);
f1 = tree_face_quadrant_corner_face (side1->quad, side1->corner);
fc1 = p4est_corner_face_corners[side1->corner][f1];
P4EST_ASSERT (0 <= fc1 && fc1 < P4EST_HALF);
tree1 = p4est_tree_array_index (trees, side1->treeid);
qid1 = side1->quadid + (side1->is_ghost ? mesh->local_num_quadrants
: tree1->quadrants_offset);
visited[j] = 1;
continue;
}
/* Examine a potential second side */
P4EST_ASSERT (side2 == NULL);
side2 =
(p4est_iter_corner_side_t *) sc_array_index_int (&info->sides, j);
P4EST_ASSERT (side2->quad != NULL);
f2 = tree_face_quadrant_corner_face (side2->quad, side2->corner);
if (side1->treeid == side2->treeid && f1 == f2) {
/* Periodicity allows for equal trees and unequal faces */
side2 = NULL;
continue;
}
/* This side as in the opposite tree */
fc2 = p4est_corner_face_corners[side2->corner][f2];
P4EST_ASSERT (0 <= fc2 && fc2 < P4EST_HALF);
code = conn->tree_to_face[P4EST_FACES * side1->treeid + f1];
orientation = code / P4EST_FACES;
P4EST_ASSERT (f2 == code % P4EST_FACES);
#ifdef P4_TO_P8
pref = p8est_face_permutation_refs[f1][f2];
pset = p8est_face_permutation_sets[pref][orientation];
diagonal = (p8est_face_permutations[pset][fc1] ^ fc2) == 3;
#else
diagonal = (fc1 ^ fc2) != orientation;
#endif
if (!diagonal) {
side2 = NULL;
continue;
}
/* We have found a diagonally opposite second side */
tree2 = p4est_tree_array_index (trees, side2->treeid);
qid2 = side2->quadid + (side2->is_ghost ? mesh->local_num_quadrants
: tree2->quadrants_offset);
if (!side1->is_ghost) {
P4EST_ASSERT (0 <= qid1 && qid1 < mesh->local_num_quadrants);
P4EST_ASSERT (mesh->quad_to_corner[P4EST_CHILDREN * qid1 +
side1->corner] == -1);
cornerid = mesh_corner_allocate (mesh, 1, &pcquad, &pccorner);
mesh->quad_to_corner[P4EST_CHILDREN * qid1 + side1->corner] =
cornerid_offset + cornerid;
*pcquad = qid2;
*pccorner = side2->corner;
}
if (!side2->is_ghost) {
P4EST_ASSERT (0 <= qid2 && qid2 < mesh->local_num_quadrants);
P4EST_ASSERT (mesh->quad_to_corner[P4EST_CHILDREN * qid2 +
side2->corner] == -1);
cornerid = mesh_corner_allocate (mesh, 1, &pcquad, &pccorner);
mesh->quad_to_corner[P4EST_CHILDREN * qid2 + side2->corner] =
cornerid_offset + cornerid;
*pcquad = qid1;
*pccorner = side1->corner;
}
visited[j] = 1;
break;
}
P4EST_ASSERT (side1 != NULL && side2 != NULL);
}
return;
}
#ifdef P4_TO_P8
if (info->tree_boundary == P8EST_CONNECT_EDGE) {
/* Tree corner neighbors across an edge are not implemented: set to -2 */
for (zz = 0; zz < cz; ++zz) {
side1 = (p4est_iter_corner_side_t *) sc_array_index (&info->sides, zz);
if (!side1->is_ghost) {
tree1 = p4est_tree_array_index (trees, side1->treeid);
qid1 = side1->quadid + tree1->quadrants_offset;
P4EST_ASSERT (0 <= qid1 && qid1 < mesh->local_num_quadrants);
P4EST_ASSERT (mesh->quad_to_corner[P4EST_CHILDREN * qid1 +
side1->corner] == -1);
mesh->quad_to_corner[P4EST_CHILDREN * qid1 + side1->corner] = -2;
}
}
return;
}
#endif
if (info->tree_boundary == P4EST_CONNECT_CORNER) {
int c1, ncorner[P4EST_DIM];
int nface[P4EST_DIM];
int ignore;
size_t z2;
int8_t *ccorners;
p4est_topidx_t t1, ntree[P4EST_DIM];
p4est_locidx_t goodones;
p4est_locidx_t *cquads;
/* Loop through all corner sides, that is the quadrants touching it. For
* each of these quadrants, determine the corner sides that can potentially
* occur by being a face neighbor as well. Exclude these face neighbors
* and the quadrant itself, record all others as corner neighbors.
*/
cquads = P4EST_ALLOC (p4est_locidx_t, cz - 1);
ccorners = P4EST_ALLOC (int8_t, cz - 1);
for (zz = 0; zz < cz; ++zz) {
side1 = (p4est_iter_corner_side_t *) sc_array_index (&info->sides, zz);
if (!side1->is_ghost) {
/* We only create corner information for processor-local quadrants */
t1 = side1->treeid;
c1 = (int) side1->corner;
tree1 = p4est_tree_array_index (trees, t1);
qid1 = side1->quadid + tree1->quadrants_offset;
P4EST_ASSERT (0 <= qid1 && qid1 < mesh->local_num_quadrants);
P4EST_ASSERT (mesh->quad_to_corner[P4EST_CHILDREN * qid1 + c1] == -1);
/* Check all quadrant faces that touch this corner */
for (i = 0; i < P4EST_DIM; ++i) {
f1 = p4est_corner_faces[c1][i];
ntree[i] = conn->tree_to_tree[P4EST_FACES * t1 + f1];
nface[i] = conn->tree_to_face[P4EST_FACES * t1 + f1];
if (ntree[i] == t1 && nface[i] == f1) {
/* This is a physical face boundary, no face neighbor present */
ncorner[i] = -1;
continue;
}
/* We have a face neighbor */
orientation = nface[i] / P4EST_FACES;
nface[i] %= P4EST_FACES;
ncorner[i] = p4est_connectivity_face_neighbor_corner
(c1, f1, nface[i], orientation);
}
/* Go through corner neighbors and collect the true corners */
goodones = 0;
for (z2 = 0; z2 < cz; ++z2) {
if (z2 == zz) {
/* We do not count ourselves as a neighbor */
continue;
}
ignore = 0;
side2 =
(p4est_iter_corner_side_t *) sc_array_index (&info->sides, z2);
P4EST_ASSERT (side2->corner >= 0);
for (i = 0; i < P4EST_DIM; ++i) {
/* Ignore if this is one of the face neighbors' corners */
if (ncorner[i] == (int) side2->corner &&
ntree[i] == side2->treeid) {
ignore = 1;
break;
}
}
if (ignore) {
continue;
}
/* Record this corner neighbor */
tree2 = p4est_tree_array_index (trees, side2->treeid);
qid2 = side2->quadid + (side2->is_ghost ? mesh->local_num_quadrants
: tree2->quadrants_offset);
cquads[goodones] = qid2;
ccorners[goodones] = (int) side2->corner;
++goodones;
}
P4EST_ASSERT ((size_t) goodones < cz);
if (goodones == 0) {
continue;
}
/* Allocate and fill corner information in the mesh structure */
cornerid = mesh_corner_allocate (mesh, goodones, &pcquad, &pccorner);
mesh->quad_to_corner[P4EST_CHILDREN * qid1 + c1] =
cornerid_offset + cornerid;
memcpy (pcquad, cquads, goodones * sizeof (p4est_locidx_t));
memcpy (pccorner, ccorners, goodones * sizeof (int8_t));
}
}
P4EST_FREE (cquads);
P4EST_FREE (ccorners);
return;
}
/* Process a corner inside the tree in pairs of diagonal neighbors */
P4EST_ASSERT (!info->tree_boundary);
side1 = (p4est_iter_corner_side_t *) sc_array_index (&info->sides, 0);
tree1 = p4est_tree_array_index (trees, side1->treeid);
qoffset = tree1->quadrants_offset;
memset (visited, 0, P4EST_CHILDREN * sizeof (int));
for (i = 0; i < P4EST_HALF; ++i) {
side1 = side2 = NULL;
qid1 = -3;
for (j = 0; j < P4EST_CHILDREN; ++j) {
if (visited[j]) {
continue;
}
/* Remember the first side we want to pair up */
if (side1 == NULL) {
side1 =
(p4est_iter_corner_side_t *) sc_array_index_int (&info->sides, j);
qid1 = side1->quadid +
(side1->is_ghost ? mesh->local_num_quadrants : qoffset);
visited[j] = 1;
continue;
}
/* Examine a potential second side */
P4EST_ASSERT (side2 == NULL);
side2 =
(p4est_iter_corner_side_t *) sc_array_index_int (&info->sides, j);
P4EST_ASSERT (side1->treeid == side2->treeid);
if (side1->corner + side2->corner != P4EST_CHILDREN - 1) {
side2 = NULL;
continue;
}
/* We have found a diagonally opposite second side */
qid2 = side2->quadid +
(side2->is_ghost ? mesh->local_num_quadrants : qoffset);
if (!side1->is_ghost) {
P4EST_ASSERT (0 <= qid1 && qid1 < mesh->local_num_quadrants);
P4EST_ASSERT (mesh->quad_to_corner[P4EST_CHILDREN * qid1 +
side1->corner] == -1);
mesh->quad_to_corner[P4EST_CHILDREN * qid1 + side1->corner] = qid2;
}
if (!side2->is_ghost) {
P4EST_ASSERT (0 <= qid2 && qid2 < mesh->local_num_quadrants);
P4EST_ASSERT (mesh->quad_to_corner[P4EST_CHILDREN * qid2 +
side2->corner] == -1);
mesh->quad_to_corner[P4EST_CHILDREN * qid2 + side2->corner] = qid1;
}
visited[j] = 1;
break;
}
P4EST_ASSERT (side1 != NULL && side2 != NULL);
}
}
p6est_lnodes_t *
p6est_lnodes_new (p6est_t * p6est, p6est_ghost_t * ghost, int degree)
{
p6est_lnodes_t *lnodes;
p6est_profile_t *profile;
p4est_lnodes_t *clnodes;
int nperelem = (degree + 1) * (degree + 1) * (degree + 1);
/* int nperface = (degree - 1) * (degree - 1); */
/* int nperedge = (degree - 1); */
p4est_locidx_t ncid, cid, enid, *en;
p4est_locidx_t nnodecols;
p4est_locidx_t nelemcols;
p4est_locidx_t nll;
p4est_locidx_t nlayers;
p4est_locidx_t *layernodecount;
p4est_locidx_t *layernodeoffsets;
p4est_locidx_t (*lr)[2];
p4est_locidx_t ncolnodes;
p4est_locidx_t *global_owned_count;
p4est_locidx_t num_owned, num_local;
p4est_gloidx_t gnum_owned, offset;
p4est_gloidx_t *owned_offsets;
int i, j, k;
int mpisize = p6est->mpisize;
int mpiret;
sc_array_t lnoview;
size_t zz, nsharers;
int Nrp = degree + 1;
if (degree == 1) {
p4est_locidx_t eid, nid, enid2, nid2;
p4est_locidx_t *newnum, newlocal, newowned;
P4EST_GLOBAL_PRODUCTION ("Into adapt p6est_lnodes_new for degree = 1\n");
p4est_log_indent_push ();
/* adapt 2 to 1 */
lnodes = p6est_lnodes_new (p6est, ghost, 2);
nll = p6est->layers->elem_count;
num_local = lnodes->num_local_nodes;
num_owned = lnodes->owned_count;
en = lnodes->element_nodes;
newnum = P4EST_ALLOC (p4est_locidx_t, P8EST_INSUL * nll);
memset (newnum, -1, P8EST_INSUL * nll * sizeof (p4est_locidx_t));
for (enid = 0, eid = 0; eid < nll; eid++) {
for (k = 0; k < 3; k++) {
for (j = 0; j < 3; j++) {
for (i = 0; i < 3; i++, enid++) {
if (k != 1 && j != 1 && i != 1) {
newnum[en[enid]] = 0;
}
}
}
}
}
newlocal = 0;
newowned = 0;
for (nid = 0; nid < num_local; nid++) {
if (newnum[nid] >= 0) {
newnum[nid] = newlocal++;
if (nid < num_owned) {
newowned++;
}
}
}
/* compress en */
enid2 = 0;
for (enid = 0, eid = 0; eid < nll; eid++) {
for (k = 0; k < 3; k++) {
for (j = 0; j < 3; j++) {
for (i = 0; i < 3; i++, enid++) {
if (k != 1 && j != 1 && i != 1) {
en[enid2++] = newnum[en[enid]];
}
}
}
}
}
P4EST_ASSERT (enid2 == P8EST_CHILDREN * nll);
lnodes->element_nodes =
P4EST_REALLOC (en, p4est_locidx_t, P8EST_CHILDREN * nll);
owned_offsets = P4EST_ALLOC (p4est_gloidx_t, mpisize + 1);
mpiret = sc_MPI_Allgather (&newowned, 1, P4EST_MPI_LOCIDX,
lnodes->global_owned_count, 1,
P4EST_MPI_LOCIDX, p6est->mpicomm);
owned_offsets[0] = 0;
for (i = 0; i < mpisize; i++) {
owned_offsets[i + 1] = owned_offsets[i] + lnodes->global_owned_count[i];
}
lnodes->global_offset = owned_offsets[p6est->mpirank];
lnodes->num_local_nodes = newlocal;
lnodes->owned_count = newowned;
lnodes->degree = 1;
lnodes->vnodes = P8EST_CHILDREN;
lnodes->nonlocal_nodes =
P4EST_REALLOC (lnodes->nonlocal_nodes, p4est_gloidx_t,
newlocal - newowned);
nsharers = lnodes->sharers->elem_count;
for (zz = 0; zz < nsharers; zz++) {
size_t nshared, zy, zw;
p6est_lnodes_rank_t *rank = p6est_lnodes_rank_array_index
(lnodes->sharers, zz);
if (rank->owned_count) {
if (rank->rank != p6est->mpirank) {
p4est_locidx_t newrankowned = 0;
p4est_locidx_t newrankoffset = -1;
for (nid = rank->owned_offset; nid < rank->owned_offset +
rank->owned_count; nid++) {
if (newnum[nid] >= 0) {
lnodes->nonlocal_nodes[newnum[nid] - newowned] =
owned_offsets[rank->rank];
newrankowned++;
if (newrankoffset < 0) {
newrankoffset = newnum[nid];
}
}
}
rank->owned_offset = newrankoffset;
rank->owned_count = newrankowned;
}
else {
rank->owned_offset = 0;
rank->owned_count = newowned;
}
}
rank->shared_mine_count = 0;
rank->shared_mine_offset = -1;
zw = 0;
nshared = rank->shared_nodes.elem_count;
for (zy = 0; zy < nshared; zy++) {
nid = *((p4est_locidx_t *) sc_array_index (&rank->shared_nodes, zy));
if (newnum[nid] >= 0) {
p4est_locidx_t *lp;
lp = (p4est_locidx_t *) sc_array_index (&rank->shared_nodes, zw++);
*lp = newnum[nid];
if (newnum[nid] < newowned) {
rank->shared_mine_count++;
if (rank->shared_mine_offset == -1) {
rank->shared_mine_offset = zw - 1;
}
}
}
}
sc_array_resize (&rank->shared_nodes, zw);
}
/* send local numbers to others */
{
sc_array_t view;
sc_array_init_data (&view, newnum, sizeof (p4est_locidx_t), newlocal);
p6est_lnodes_share_owned (&view, lnodes);
}
nid2 = 0;
for (nid = num_owned; nid < num_local; nid++) {
if (newnum[nid] >= 0) {
lnodes->nonlocal_nodes[nid2++] += (p4est_gloidx_t) newnum[nid];
}
}
P4EST_ASSERT (nid2 == newlocal - newowned);
P4EST_FREE (owned_offsets);
P4EST_FREE (newnum);
p4est_log_indent_pop ();
P4EST_GLOBAL_PRODUCTION ("Done adapt p6est_lnodes_new for degree = 1\n");
return lnodes;
}
P4EST_GLOBAL_PRODUCTION ("Into p6est_lnodes_new\n");
p4est_log_indent_push ();
P4EST_ASSERT (degree >= 1);
lnodes = P4EST_ALLOC (p6est_lnodes_t, 1);
/* first get the profile */
profile = p6est_profile_new_local (p6est, ghost, P6EST_PROFILE_INTERSECTION,
P8EST_CONNECT_FULL, degree);
p6est_profile_sync (profile);
lr = (p4est_locidx_t (*)[2]) profile->lnode_ranges;
clnodes = profile->lnodes;
nnodecols = clnodes->num_local_nodes;
nelemcols = clnodes->num_local_elements;
en = clnodes->element_nodes;
layernodecount = P4EST_ALLOC_ZERO (p4est_locidx_t, nnodecols);
layernodeoffsets = P4EST_ALLOC_ZERO (p4est_locidx_t, nnodecols + 1);
for (cid = 0, enid = 0; cid < nelemcols; cid++) {
for (j = 0; j < Nrp; j++) {
for (i = 0; i < Nrp; i++, enid++) {
ncid = en[enid];
nlayers = lr[ncid][1];
P4EST_ASSERT (nlayers);
ncolnodes = nlayers * degree + 1;
layernodecount[ncid] = ncolnodes;
}
}
}
num_owned = 0;
num_local = 0;
for (ncid = 0; ncid < nnodecols; ncid++) {
num_local += layernodecount[ncid];
if (ncid < clnodes->owned_count) {
num_owned += layernodecount[ncid];
}
}
P4EST_VERBOSEF ("p6est_lnodes: %d owned %d local\n", num_owned, num_local);
if (nnodecols) {
layernodeoffsets[0] = 0;
for (ncid = 0; ncid < nnodecols; ncid++) {
layernodeoffsets[ncid + 1] = layernodeoffsets[ncid] +
layernodecount[ncid];
}
}
gnum_owned = num_owned;
owned_offsets = P4EST_ALLOC (p4est_gloidx_t, mpisize + 1);
global_owned_count = P4EST_ALLOC (p4est_locidx_t, mpisize);
mpiret = sc_MPI_Allgather (&gnum_owned, 1, P4EST_MPI_GLOIDX,
owned_offsets, 1, P4EST_MPI_GLOIDX,
p6est->mpicomm);
SC_CHECK_MPI (mpiret);
offset = 0;
for (i = 0; i < mpisize; i++) {
global_owned_count[i] = (p4est_locidx_t) owned_offsets[i];
gnum_owned = owned_offsets[i];
owned_offsets[i] = offset;
offset += gnum_owned;
}
owned_offsets[mpisize] = offset;
nll = p6est->layers->elem_count;
nsharers = clnodes->sharers->elem_count;
lnodes->mpicomm = p6est->mpicomm;
lnodes->num_local_nodes = num_local;
lnodes->owned_count = num_owned;
lnodes->global_offset = owned_offsets[p6est->mpirank];
lnodes->nonlocal_nodes =
P4EST_ALLOC (p4est_gloidx_t, num_local - num_owned);
lnodes->sharers =
sc_array_new_size (sizeof (p6est_lnodes_rank_t), nsharers);
lnodes->global_owned_count = global_owned_count;
lnodes->degree = degree;
lnodes->vnodes = nperelem;
lnodes->num_local_elements = nll;
lnodes->face_code = P4EST_ALLOC (p6est_lnodes_code_t, nll);
lnodes->element_nodes = P4EST_ALLOC (p4est_locidx_t, nperelem * nll);
p6est_profile_element_to_node (p6est, profile, layernodeoffsets,
lnodes->element_nodes, lnodes->face_code);
for (zz = 0; zz < nsharers; zz++) {
p4est_lnodes_rank_t *crank = p4est_lnodes_rank_array_index
(clnodes->sharers, zz);
p6est_lnodes_rank_t *rank = p6est_lnodes_rank_array_index
(lnodes->sharers, zz);
size_t zy;
size_t nshared;
rank->rank = crank->rank;
sc_array_init (&rank->shared_nodes, sizeof (p4est_locidx_t));
nshared = crank->shared_nodes.elem_count;
rank->owned_offset = -1;
rank->owned_count = 0;
rank->shared_mine_count = 0;
rank->shared_mine_offset = -1;
for (zy = 0; zy < nshared; zy++) {
p4est_locidx_t cnid =
*((p4est_locidx_t *) sc_array_index (&crank->shared_nodes, zy));
p4est_locidx_t *lp;
p4est_locidx_t nthis, il;
p4est_locidx_t old_count = rank->shared_nodes.elem_count;
nthis = layernodecount[cnid];
lp =
(p4est_locidx_t *) sc_array_push_count (&rank->shared_nodes, nthis);
for (il = 0; il < nthis; il++) {
lp[il] = layernodeoffsets[cnid] + il;
if (zy >= (size_t) crank->shared_mine_offset
&& (p4est_locidx_t) zy - crank->shared_mine_offset <
crank->shared_mine_count) {
rank->shared_mine_count++;
if (rank->shared_mine_offset == -1) {
rank->shared_mine_offset = old_count + il;
}
}
if (cnid >= crank->owned_offset
&& cnid - crank->owned_offset < crank->owned_count) {
rank->owned_count++;
if (rank->owned_offset == -1) {
rank->owned_offset = lp[il];
}
}
}
}
if (rank->rank == p6est->mpirank) {
rank->owned_offset = 0;
rank->owned_count = num_owned;
}
}
memcpy (layernodecount, layernodeoffsets,
nnodecols * sizeof (p4est_locidx_t));
sc_array_init_data (&lnoview, layernodecount, sizeof (p4est_locidx_t),
(size_t) nnodecols);
p4est_lnodes_share_owned (&lnoview, clnodes);
for (zz = 0; zz < nsharers; zz++) {
p4est_lnodes_rank_t *crank = p4est_lnodes_rank_array_index
(clnodes->sharers, zz);
if (crank->rank == p6est->mpirank) {
continue;
}
for (ncid = crank->owned_offset;
ncid < crank->owned_offset + crank->owned_count; ncid++) {
p4est_gloidx_t owners_offset;
p4est_locidx_t nid;
P4EST_ASSERT (ncid >= clnodes->owned_count);
owners_offset = owned_offsets[crank->rank] + layernodecount[ncid];
for (nid = layernodeoffsets[ncid]; nid < layernodeoffsets[ncid + 1];
nid++) {
P4EST_ASSERT (nid >= num_owned);
P4EST_ASSERT (nid < num_local);
lnodes->nonlocal_nodes[nid - num_owned] = owners_offset++;
}
}
}
p6est_profile_destroy (profile);
P4EST_FREE (owned_offsets);
P4EST_FREE (layernodecount);
P4EST_FREE (layernodeoffsets);
p4est_log_indent_pop ();
P4EST_GLOBAL_PRODUCTION ("Done p6est_lnodes_new\n");
return lnodes;
}
p4est_gloidx_t *
p6est_lnodes_get_column_labels (p6est_t * p6est, p8est_lnodes_t * lnodes)
{
p4est_gloidx_t *labels;
p4est_gloidx_t num_cols = 0;
p4est_gloidx_t global_num_cols = 0;
p4est_topidx_t jt;
p4est_tree_t *tree;
sc_array_t *tquadrants;
p4est_quadrant_t *col;
size_t zz, first, last;
p4est_locidx_t lfirst, llast, lk;
int stride = lnodes->degree + 1;
int vnodes = lnodes->vnodes;
int mpiret, i;
labels = P4EST_ALLOC (p4est_gloidx_t, lnodes->owned_count);
memset (labels, -1, lnodes->owned_count * sizeof (*labels));
for (jt = p6est->columns->first_local_tree;
jt <= p6est->columns->last_local_tree; ++jt) {
tree = p4est_tree_array_index (p6est->columns->trees, jt);
tquadrants = &tree->quadrants;
for (zz = 0; zz < tquadrants->elem_count; ++zz) {
col = p4est_quadrant_array_index (tquadrants, zz);
P6EST_COLUMN_GET_RANGE (col, &first, &last);
lfirst = (p4est_locidx_t) first;
llast = (p4est_locidx_t) last;
for (i = 0; i < vnodes; i += stride) {
p4est_locidx_t fnid = lnodes->element_nodes[vnodes * lfirst + i];
p4est_locidx_t lnid =
lnodes->element_nodes[vnodes * (llast - 1) + i + (stride - 1)];
P4EST_ASSERT (lnid >= 0);
P4EST_ASSERT (lnid >= fnid);
P4EST_ASSERT (fnid < lnodes->num_local_nodes);
if (lnid < lnodes->owned_count) {
P4EST_ASSERT (fnid < lnodes->owned_count);
if (labels[fnid] < 0) {
for (lk = fnid; lk <= lnid; lk++) {
labels[lk] = num_cols;
}
num_cols++;
}
}
}
}
}
mpiret =
sc_MPI_Exscan (&num_cols, &global_num_cols, 1, P4EST_MPI_GLOIDX,
sc_MPI_SUM, lnodes->mpicomm);
SC_CHECK_MPI (mpiret);
if (!p6est->mpirank) {
global_num_cols = 0;
}
for (lk = 0; lk < lnodes->owned_count; lk++) {
labels[lk] += global_num_cols;
}
#if 0
{
sc_array_t view;
sc_array_init_data (&view, labels, sizeof (*labels),
(size_t) lnodes->num_local_nodes);
p6est_lnodes_share_owned (&view, lnodes);
}
#endif
for (lk = 0; lk < lnodes->owned_count; lk++) {
P4EST_ASSERT (labels[lk] >= 0);
}
return labels;
}
void
problem_init
(
int argc,
char* argv [],
p4est_t* p4est,
p4est_geometry_t* p4est_geom,
dgmath_jit_dbase_t* dgmath_jit_dbase,
int proc_size,
sc_MPI_Comm mpicomm,
int load_from_checkpoint
)
{
mpi_assert((P4EST_DIM) == 2 || (P4EST_DIM) == 3);
int world_rank, world_size;
sc_MPI_Comm_rank(sc_MPI_COMM_WORLD, &world_rank);
sc_MPI_Comm_size(sc_MPI_COMM_WORLD, &world_size);
double* Au = P4EST_ALLOC_ZERO(double, 1);
double* rhs = P4EST_ALLOC_ZERO(double, 1);
double* u = P4EST_ALLOC_ZERO(double, 1);
double* f = P4EST_ALLOC_ZERO(double, 1);
double* u_analytic = P4EST_ALLOC_ZERO(double, 1);
int local_nodes = 1;
problem_input_t input = problem_input("options.input");
int endlevel = input.endlevel;
int degree = input.degree;
ip_flux_params_t ip_flux_params;
ip_flux_params.ip_flux_penalty_prefactor = input.ip_flux_penalty;
ip_flux_params.ip_flux_penalty_calculate_fcn = sipg_flux_vector_calc_penalty_maxp2_over_minh;
p4est_ghost_t* ghost = p4est_ghost_new (p4est, P4EST_CONNECT_FACE);
/* create space for storing the ghost data */
element_data_t* ghost_data = P4EST_ALLOC (element_data_t,
ghost->ghosts.elem_count);
p4est_partition(p4est, 0, NULL);
p4est_balance (p4est, P4EST_CONNECT_FACE, NULL);
grid_fcn_t boundary_flux_fcn = zero_fcn;
problem_data_t prob_vecs;
prob_vecs.rhs = rhs;
prob_vecs.Au = Au;
prob_vecs.u = u;
prob_vecs.f = f;
prob_vecs.local_nodes = local_nodes;
for (int level = 0; level < endlevel; ++level){
if (level != 0){
p4est_refine_ext(p4est,
0,
-1,
refine_function,
NULL,
NULL
);
p4est_balance_ext
(
p4est,
P4EST_CONNECT_FACE,
NULL,
NULL
);
p4est_ghost_destroy(ghost);
P4EST_FREE(ghost_data);
ghost = p4est_ghost_new(p4est, P4EST_CONNECT_FACE);
ghost_data = P4EST_ALLOC(element_data_t, ghost->ghosts.elem_count);
}
}
p4est_partition(p4est, 1, NULL);
p4est_balance_ext
(
p4est,
P4EST_CONNECT_FACE,
NULL,
NULL
);
p4est_ghost_destroy(ghost);
P4EST_FREE(ghost_data);
ghost = p4est_ghost_new(p4est, P4EST_CONNECT_FACE);
ghost_data = P4EST_ALLOC(element_data_t, ghost->ghosts.elem_count);
weakeqn_ptrs_t prob_fcns;
prob_fcns.apply_lhs = problem_apply_aij;
element_data_init(p4est, degree);
local_nodes = element_data_get_local_nodes(p4est);
printf("RANK %d: Nodes = %d\n", world_rank, local_nodes);
Au = P4EST_REALLOC(Au, double, local_nodes);
u = P4EST_REALLOC(u, double, local_nodes);
f = P4EST_REALLOC(f, double, local_nodes);
rhs = P4EST_REALLOC(rhs, double, local_nodes);
u_analytic = P4EST_REALLOC(u_analytic, double, local_nodes);
prob_vecs.Au = Au;
prob_vecs.u = u;
prob_vecs.f = f;
prob_vecs.rhs = rhs;
prob_vecs.local_nodes = local_nodes;
linalg_fill_vec(u, 0., local_nodes);
element_data_init_node_vec(p4est,f,f_fcn,dgmath_jit_dbase);
prob_vecs.vector_flux_fcn_data
= sipg_flux_vector_dirichlet_fetch_fcns
(
boundary_fcn,
&ip_flux_params
);
prob_vecs.scalar_flux_fcn_data
= sipg_flux_scalar_dirichlet_fetch_fcns
(
boundary_flux_fcn
);
problem_build_rhs
(
p4est,
&prob_vecs,
&prob_fcns,
ghost,
ghost_data,
dgmath_jit_dbase
);
clock_t begin = 0;
clock_t end = -1;
if (world_rank == 0){
begin = clock();
}
int vcycle_iter = 3;
double vcycle_rtol = 1e-3;
double vcycle_atol = 0.;
int smooth_iter = 8;
int cg_eigs_iter = 10;
double max_eig_factor = 1.1;
int max_eig_reuse = 1;
double lmax_lmin_rat = 30.;
int coarse_iter = 100;
double coarse_rtol = 1e-8;
int save_vtk_snapshot = 0;
int perform_checksum = 0;
multigrid_data_t* mg_data
= multigrid_data_init
(
world_rank,
endlevel,
vcycle_iter,
vcycle_rtol,
vcycle_atol,
smooth_iter,
cg_eigs_iter,
max_eig_factor,
max_eig_reuse,
lmax_lmin_rat,
CG,
coarse_iter,
coarse_rtol,
save_vtk_snapshot,
perform_checksum,
RES_AND_EIG_LOG,
dgmath_jit_dbase
);
multigrid_solve
(
p4est,
&prob_vecs,
&prob_fcns,
mg_data,
&ghost,
&ghost_data
);
multigrid_data_destroy(mg_data);
element_data_init_node_vec(p4est, u_analytic, analytic_solution_fcn, dgmath_jit_dbase);
linalg_vec_axpy(-1., u, u_analytic, local_nodes);
double local_l2_norm_sqr = element_data_compute_l2_norm_sqr_no_local
(
p4est,
u_analytic,
dgmath_jit_dbase
);
double local_nodes_dbl = (double)local_nodes;
double local_reduce [2];
local_reduce[0] = local_nodes_dbl;
local_reduce[1] = local_l2_norm_sqr;
double global_reduce [2];
sc_reduce
(
&local_reduce[0],
&global_reduce[0],
2,
sc_MPI_DOUBLE,
sc_MPI_SUM,
0,
sc_MPI_COMM_WORLD
);
double global_nodes_dbl = global_reduce[0];
double global_l2_norm_sqr = global_reduce[1];
if (world_rank == 0){
end = clock();
double time_spent = (double)(end - begin) / CLOCKS_PER_SEC;
printf
(
"\n\n[HP_AMR]: %d %d %d %.25f %f \n\n",
degree,
(int)p4est->global_num_quadrants,
(int)global_nodes_dbl,
sqrt(global_l2_norm_sqr),
/* info.iterations, */
/* info.residual_norm, */
time_spent
);
}
if (ghost) {
p4est_ghost_destroy (ghost);
P4EST_FREE (ghost_data);
ghost = NULL;
ghost_data = NULL;
}
P4EST_FREE(f);
P4EST_FREE(Au);
P4EST_FREE(rhs);
P4EST_FREE(u_analytic);
P4EST_FREE(u);
}
/** Timestep the advection problem.
*
* Update the state, refine, repartition, and write the solution to file.
*
* \param [in,out] p4est the forest, whose state is updated
* \param [in] time the end time
*/
static void
step3_timestep (p4est_t * p4est, double time)
{
double t = 0.;
double dt = 0.;
int i;
step3_data_t *ghost_data;
step3_ctx_t *ctx = (step3_ctx_t *) p4est->user_pointer;
int refine_period = ctx->refine_period;
int repartition_period = ctx->repartition_period;
int write_period = ctx->write_period;
int recursive = 0;
int allowed_level = P4EST_QMAXLEVEL;
int allowcoarsening = 1;
int callbackorphans = 0;
int mpiret;
double orig_max_err = ctx->max_err;
double umax, global_umax;
p4est_ghost_t *ghost;
/* create the ghost quadrants */
ghost = p4est_ghost_new (p4est, P4EST_CONNECT_FULL);
/* create space for storing the ghost data */
ghost_data = P4EST_ALLOC (step3_data_t, ghost->ghosts.elem_count);
/* synchronize the ghost data */
p4est_ghost_exchange_data (p4est, ghost, ghost_data);
/* initialize du/dx estimates */
p4est_iterate (p4est, ghost, (void *) ghost_data, /* pass in ghost data that we just exchanged */
step3_reset_derivatives, /* blank the previously calculated derivatives */
step3_minmod_estimate, /* compute the minmod estimate of each cell's derivative */
#ifdef P4_TO_P8
NULL, /* there is no callback for the edges between quadrants */
#endif
NULL); /* there is no callback for the corners between quadrants */
for (t = 0., i = 0; t < time; t += dt, i++) {
P4EST_GLOBAL_PRODUCTIONF ("time %f\n", t);
/* refine */
if (!(i % refine_period)) {
if (i) {
/* compute umax */
umax = 0.;
/* initialize derivative estimates */
p4est_iterate (p4est, NULL, (void *) &umax, /* pass in ghost data that we just exchanged */
step3_compute_max, /* blank the previously calculated derivatives */
NULL, /* there is no callback for the faces between quadrants */
#ifdef P4_TO_P8
NULL, /* there is no callback for the edges between quadrants */
#endif
NULL); /* there is no callback for the corners between quadrants */
mpiret =
sc_MPI_Allreduce (&umax, &global_umax, 1, sc_MPI_DOUBLE, sc_MPI_MAX,
p4est->mpicomm);
SC_CHECK_MPI (mpiret);
ctx->max_err = orig_max_err * global_umax;
P4EST_GLOBAL_PRODUCTIONF ("u_max %f\n", global_umax);
/* adapt */
p4est_refine_ext (p4est, recursive, allowed_level,
step3_refine_err_estimate, NULL,
step3_replace_quads);
p4est_coarsen_ext (p4est, recursive, callbackorphans,
step3_coarsen_err_estimate, NULL,
step3_replace_quads);
p4est_balance_ext (p4est, P4EST_CONNECT_FACE, NULL,
step3_replace_quads);
p4est_ghost_destroy (ghost);
P4EST_FREE (ghost_data);
ghost = NULL;
ghost_data = NULL;
}
dt = step3_get_timestep (p4est);
}
/* repartition */
if (i && !(i % repartition_period)) {
p4est_partition (p4est, allowcoarsening, NULL);
if (ghost) {
p4est_ghost_destroy (ghost);
P4EST_FREE (ghost_data);
ghost = NULL;
ghost_data = NULL;
}
}
/* write out solution */
if (!(i % write_period)) {
step3_write_solution (p4est, i);
}
/* synchronize the ghost data */
if (!ghost) {
ghost = p4est_ghost_new (p4est, P4EST_CONNECT_FULL);
ghost_data = P4EST_ALLOC (step3_data_t, ghost->ghosts.elem_count);
p4est_ghost_exchange_data (p4est, ghost, ghost_data);
}
/* compute du/dt */
/* *INDENT-OFF* */
p4est_iterate (p4est, /* the forest */
ghost, /* the ghost layer */
(void *) ghost_data, /* the synchronized ghost data */
step3_quad_divergence, /* callback to compute each quad's
interior contribution to du/dt */
step3_upwind_flux, /* callback to compute each quads'
faces' contributions to du/du */
#ifdef P4_TO_P8
NULL, /* there is no callback for the
edges between quadrants */
#endif
NULL); /* there is no callback for the
corners between quadrants */
/* *INDENT-ON* */
/* update u */
p4est_iterate (p4est, NULL, /* ghosts are not needed for this loop */
(void *) &dt, /* pass in dt */
step3_timestep_update, /* update each cell */
NULL, /* there is no callback for the faces between quadrants */
#ifdef P4_TO_P8
NULL, /* there is no callback for the edges between quadrants */
#endif
NULL); /* there is no callback for the corners between quadrants */
/* synchronize the ghost data */
p4est_ghost_exchange_data (p4est, ghost, ghost_data);
/* update du/dx estimate */
p4est_iterate (p4est, ghost, (void *) ghost_data, /* pass in ghost data that we just exchanged */
step3_reset_derivatives, /* blank the previously calculated derivatives */
step3_minmod_estimate, /* compute the minmod estimate of each cell's derivative */
#ifdef P4_TO_P8
NULL, /* there is no callback for the edges between quadrants */
#endif
NULL); /* there is no callback for the corners between quadrants */
}
P4EST_FREE (ghost_data);
p4est_ghost_destroy (ghost);
}
int
main (int argc, char **argv)
{
int rank;
int num_procs;
int mpiret;
sc_MPI_Comm mpicomm;
p4est_t *p4est, *copy;
p4est_connectivity_t *connectivity;
int i;
p4est_topidx_t t;
size_t qz;
p4est_locidx_t num_quadrants_on_last;
p4est_locidx_t *num_quadrants_in_proc;
p4est_gloidx_t *pertree1, *pertree2;
p4est_quadrant_t *quad;
p4est_tree_t *tree;
user_data_t *user_data;
int64_t sum;
unsigned crc;
mpiret = sc_MPI_Init (&argc, &argv);
SC_CHECK_MPI (mpiret);
mpicomm = sc_MPI_COMM_WORLD;
mpiret = sc_MPI_Comm_rank (mpicomm, &rank);
SC_CHECK_MPI (mpiret);
sc_init (mpicomm, 1, 1, NULL, SC_LP_DEFAULT);
/* create connectivity and forest structures */
#ifdef P4_TO_P8
connectivity = p8est_connectivity_new_twocubes ();
#else
connectivity = p4est_connectivity_new_corner ();
#endif
p4est = p4est_new_ext (mpicomm, connectivity, 15, 0, 0,
sizeof (user_data_t), init_fn, NULL);
pertree1 = P4EST_ALLOC (p4est_gloidx_t, p4est->connectivity->num_trees + 1);
pertree2 = P4EST_ALLOC (p4est_gloidx_t, p4est->connectivity->num_trees + 1);
num_procs = p4est->mpisize;
num_quadrants_in_proc = P4EST_ALLOC (p4est_locidx_t, num_procs);
/* refine and balance to make the number of elements interesting */
test_pertree (p4est, NULL, pertree1);
p4est_refine (p4est, 1, refine_fn, init_fn);
test_pertree (p4est, NULL, pertree1);
/* Set an arbitrary partition.
*
* Since this is just a test we assume the global number of
* quadrants will fit in an int32_t
*/
num_quadrants_on_last = (p4est_locidx_t) p4est->global_num_quadrants;
for (i = 0; i < num_procs - 1; ++i) {
num_quadrants_in_proc[i] = (p4est_locidx_t) i + 1; /* type ok */
num_quadrants_on_last -= (p4est_locidx_t) i + 1; /* type ok */
}
num_quadrants_in_proc[num_procs - 1] = num_quadrants_on_last;
SC_CHECK_ABORT (num_quadrants_on_last > 0,
"Negative number of quadrants on the last processor");
/* Save a checksum of the original forest */
crc = p4est_checksum (p4est);
/* partition the forest */
(void) p4est_partition_given (p4est, num_quadrants_in_proc);
test_pertree (p4est, pertree1, pertree2);
/* Double check that we didn't loose any quads */
SC_CHECK_ABORT (crc == p4est_checksum (p4est),
"bad checksum, missing a quad");
/* count the actual number of quadrants per proc */
SC_CHECK_ABORT (num_quadrants_in_proc[rank]
== p4est->local_num_quadrants,
"partition failed, wrong number of quadrants");
/* check user data content */
for (t = p4est->first_local_tree; t <= p4est->last_local_tree; ++t) {
tree = p4est_tree_array_index (p4est->trees, t);
for (qz = 0; qz < tree->quadrants.elem_count; ++qz) {
quad = p4est_quadrant_array_index (&tree->quadrants, qz);
user_data = (user_data_t *) quad->p.user_data;
sum = quad->x + quad->y + quad->level;
SC_CHECK_ABORT (user_data->a == t, "bad user_data, a");
SC_CHECK_ABORT (user_data->sum == sum, "bad user_data, sum");
}
}
/* do a weighted partition with uniform weights */
p4est_partition (p4est, 0, weight_one);
test_pertree (p4est, pertree1, pertree2);
SC_CHECK_ABORT (crc == p4est_checksum (p4est),
"bad checksum after uniformly weighted partition");
/* copy the p4est */
copy = p4est_copy (p4est, 1);
SC_CHECK_ABORT (crc == p4est_checksum (copy), "bad checksum after copy");
/* do a weighted partition with many zero weights */
weight_counter = 0;
weight_index = (rank == 1) ? 1342 : 0;
p4est_partition (copy, 0, weight_once);
test_pertree (copy, pertree1, pertree2);
SC_CHECK_ABORT (crc == p4est_checksum (copy),
"bad checksum after unevenly weighted partition 1");
/* do a weighted partition with many zero weights */
weight_counter = 0;
weight_index = 0;
p4est_partition (copy, 0, weight_once);
test_pertree (copy, pertree1, pertree2);
SC_CHECK_ABORT (crc == p4est_checksum (copy),
"bad checksum after unevenly weighted partition 2");
/* do a weighted partition with many zero weights
*
* Since this is just a test we assume the local number of
* quadrants will fit in an int
*/
weight_counter = 0;
weight_index =
(rank == num_procs - 1) ? ((int) copy->local_num_quadrants - 1) : 0;
p4est_partition (copy, 0, weight_once);
test_pertree (copy, pertree1, pertree2);
SC_CHECK_ABORT (crc == p4est_checksum (copy),
"bad checksum after unevenly weighted partition 3");
/* check user data content */
for (t = copy->first_local_tree; t <= copy->last_local_tree; ++t) {
tree = p4est_tree_array_index (copy->trees, t);
for (qz = 0; qz < tree->quadrants.elem_count; ++qz) {
quad = p4est_quadrant_array_index (&tree->quadrants, qz);
user_data = (user_data_t *) quad->p.user_data;
sum = quad->x + quad->y + quad->level;
SC_CHECK_ABORT (user_data->a == t, "bad user_data, a");
SC_CHECK_ABORT (user_data->sum == sum, "bad user_data, sum");
}
}
/* Add another test. Overwrites pertree1, pertree2 */
test_partition_circle (mpicomm, connectivity, pertree1, pertree2);
/* clean up and exit */
P4EST_FREE (pertree1);
P4EST_FREE (pertree2);
P4EST_FREE (num_quadrants_in_proc);
p4est_destroy (p4est);
p4est_destroy (copy);
p4est_connectivity_destroy (connectivity);
sc_finalize ();
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
static void
test_partition_circle (sc_MPI_Comm mpicomm,
p4est_connectivity_t * connectivity,
p4est_gloidx_t * pertree1, p4est_gloidx_t * pertree2)
{
int i, j;
int num_procs;
int empty_proc1, empty_proc2;
unsigned crc1, crc2;
p4est_gloidx_t global_num;
p4est_locidx_t *new_counts;
p4est_t *p4est, *copy;
/* Create a forest and make a copy */
circle_count = 0;
p4est = p4est_new_ext (mpicomm, connectivity, 0, 3, 1,
sizeof (int), circle_init, NULL);
num_procs = p4est->mpisize;
test_pertree (p4est, NULL, pertree1);
global_num = p4est->global_num_quadrants;
crc1 = p4est_checksum (p4est);
copy = p4est_copy (p4est, 1);
P4EST_ASSERT (p4est_checksum (copy) == crc1);
new_counts = P4EST_ALLOC (p4est_locidx_t, num_procs);
/* Partition with one empty processor */
if (num_procs > 1) {
P4EST_GLOBAL_INFO ("First circle partition\n");
empty_proc1 = num_procs / 3;
j = 0;
for (i = 0; i < num_procs; ++i) {
if (i == empty_proc1) {
new_counts[i] = 0;
}
else {
new_counts[i] =
p4est_partition_cut_gloidx (global_num, j + 1, num_procs - 1) -
p4est_partition_cut_gloidx (global_num, j, num_procs - 1);
P4EST_ASSERT (new_counts[i] >= 0);
++j;
}
}
P4EST_ASSERT (j == num_procs - 1);
p4est_partition_given (p4est, new_counts);
test_pertree (p4est, pertree1, pertree2);
crc2 = p4est_checksum (p4est);
SC_CHECK_ABORT (crc1 == crc2, "First checksum mismatch");
}
/* Partition with two empty processors */
if (num_procs > 2) {
P4EST_GLOBAL_INFO ("Second circle partition\n");
empty_proc1 = (2 * num_procs) / 3 - 2;
empty_proc2 = (2 * num_procs) / 3;
j = 0;
for (i = 0; i < num_procs; ++i) {
if (i == empty_proc1 || i == empty_proc2) {
new_counts[i] = 0;
}
else {
new_counts[i] =
p4est_partition_cut_gloidx (global_num, j + 1, num_procs - 2) -
p4est_partition_cut_gloidx (global_num, j, num_procs - 2);
P4EST_ASSERT (new_counts[i] >= 0);
++j;
}
}
P4EST_ASSERT (j == num_procs - 2);
p4est_partition_given (p4est, new_counts);
test_pertree (p4est, pertree1, pertree2);
crc2 = p4est_checksum (p4est);
SC_CHECK_ABORT (crc1 == crc2, "Second checksum mismatch");
}
/* Uniform partition */
P4EST_GLOBAL_INFO ("Third circle partition\n");
p4est_partition (p4est, 0, NULL);
test_pertree (p4est, pertree1, pertree2);
crc2 = p4est_checksum (p4est);
SC_CHECK_ABORT (crc1 == crc2, "Third checksum mismatch");
SC_CHECK_ABORT (p4est_is_equal (p4est, copy, 1), "Forest mismatch");
P4EST_FREE (new_counts);
p4est_destroy (copy);
p4est_destroy (p4est);
}
int
main(int argc, char **argv)
{
int mpiret, i;
mpi_context_t mpi_context, *mpi = &mpi_context;
p4est_t *p4est;
p4est_connectivity_t *connectivity;
pchase_world_t *W;
/* initialize MPI and p4est internals */
mpiret = MPI_Init(&argc, &argv);
SC_CHECK_MPI(mpiret);
mpi->mpicomm = MPI_COMM_WORLD; /* your favourite comm here */
mpiret = MPI_Comm_size(mpi->mpicomm, &mpi->mpisize);
SC_CHECK_MPI(mpiret);
mpiret = MPI_Comm_rank(mpi->mpicomm, &mpi->mpirank);
SC_CHECK_MPI(mpiret);
/* Sets global program identifiers (e.g. the MPIrank) and some flags */
sc_init(mpi->mpicomm, 1, 1, NULL, SC_LP_SILENT);
/* Registers p4est with the SC Library and sets the logging behavior */
p4est_init(NULL, SC_LP_SILENT);
/* build up the world */
W = pchase_world_init();
/* store connectivity for a unitsquare */
connectivity = p4est_connectivity_new_unitsquare();
/* build uniform tree and get space for 25 particles each */
p4est = p4est_new_ext(mpi->mpicomm, connectivity, 0, 4, 1,
sizeof(pchase_quadrant_data_t), W->init_fn, NULL);
/* initialize everything depending on p4est */
pchase_world_init_p4est(W, p4est);
/* if (W->p4est->mpirank == 0) { */
/* for (i=0; i<100; i++) */
/*
* sc_list_append(W->particle_push_list,
* pchase_world_random_particle(W));
*/
/* } */
pchase_particle_t * p = P4EST_ALLOC(pchase_particle_t, 1);
p->x[0] = 0.5;
p->x[1] = 0.1;
sc_list_append(W->particle_push_list, p);
/* this has to be done for each proc */
pchase_world_insert_particles(W);
pchase_world_insert_particles(W);
/* let this particle move */
pchase_world_simulate(W);
#ifdef DEBUG
/* print out all quadrants */
p4est_iterate(W->p4est, NULL, NULL, W->viter_fn, NULL, NULL);
#endif
/* destroy all particles, p4est and its connectivity structure */
pchase_world_destroy(W);
p4est_destroy(p4est);
p4est_connectivity_destroy(connectivity);
/* clean up and exit */
sc_finalize();
mpiret = MPI_Finalize();
SC_CHECK_MPI(mpiret);
return 0;
}
int
p6est_vtk_write_point_scalar (p6est_t * p6est,
const char *filename,
const char *scalar_name, const double *values)
{
const int mpirank = p6est->mpirank;
const p4est_locidx_t Ncells = (p4est_locidx_t) p6est->layers->elem_count;
const p4est_locidx_t Ncorners = P8EST_CHILDREN * Ncells; /* type ok */
int retval;
p4est_locidx_t il;
#ifndef P4EST_VTK_ASCII
P4EST_VTK_FLOAT_TYPE *float_data;
#endif
char vtufilename[BUFSIZ];
FILE *vtufile;
/* Have each proc write to its own file */
snprintf (vtufilename, BUFSIZ, "%s_%04d.vtu", filename, mpirank);
/* To be able to fseek in a file you cannot open in append mode.
* so you need to open with "r+" and fseek to SEEK_END.
*/
vtufile = fopen (vtufilename, "rb+");
if (vtufile == NULL) {
P4EST_LERRORF ("Could not open %s for output\n", vtufilename);
return -1;
}
retval = fseek (vtufile, 0L, SEEK_END);
if (retval) {
P4EST_LERRORF ("Could not fseek %s for output\n", vtufilename);
fclose (vtufile);
return -1;
}
/* write point position data */
fprintf (vtufile, " <DataArray type=\"%s\" Name=\"%s\""
" format=\"%s\">\n",
P4EST_VTK_FLOAT_NAME, scalar_name, P4EST_VTK_FORMAT_STRING);
#ifdef P4EST_VTK_ASCII
for (il = 0; il < Ncorners; ++il) {
#ifdef P4EST_VTK_DOUBLES
fprintf (vtufile, " %24.16e\n", values[il]);
#else
fprintf (vtufile, " %16.8e\n", values[il]);
#endif
}
#else
float_data = P4EST_ALLOC (P4EST_VTK_FLOAT_TYPE, Ncorners);
for (il = 0; il < Ncorners; ++il) {
float_data[il] = (P4EST_VTK_FLOAT_TYPE) values[il];
}
fprintf (vtufile, " ");
/* TODO: Don't allocate the full size of the array, only allocate
* the chunk that will be passed to zlib and do this a chunk
* at a time.
*/
retval = p6est_vtk_write_binary (vtufile, (char *) float_data,
sizeof (*float_data) * Ncorners);
fprintf (vtufile, "\n");
if (retval) {
P4EST_LERROR ("p6est_vtk: Error encoding points\n");
fclose (vtufile);
return -1;
}
P4EST_FREE (float_data);
#endif
fprintf (vtufile, " </DataArray>\n");
if (ferror (vtufile)) {
P4EST_LERROR ("p6est_vtk: Error writing point scalar\n");
fclose (vtufile);
return -1;
}
if (fclose (vtufile)) {
P4EST_LERROR ("p6est_vtk: Error closing point scalar\n");
return -1;
}
vtufile = NULL;
/* Only have the root write to the parallel vtk file */
if (mpirank == 0) {
char pvtufilename[BUFSIZ];
FILE *pvtufile;
snprintf (pvtufilename, BUFSIZ, "%s.pvtu", filename);
pvtufile = fopen (pvtufilename, "ab");
if (!pvtufile) {
P4EST_LERRORF ("Could not open %s for output\n", vtufilename);
return -1;
}
fprintf (pvtufile, " <PDataArray type=\"%s\" Name=\"%s\""
" format=\"%s\"/>\n",
P4EST_VTK_FLOAT_NAME, scalar_name, P4EST_VTK_FORMAT_STRING);
if (ferror (pvtufile)) {
P4EST_LERROR ("p6est_vtk: Error writing parallel point scalar\n");
fclose (pvtufile);
return -1;
}
if (fclose (pvtufile)) {
P4EST_LERROR ("p6est_vtk: Error closing parallel point scalar\n");
return -1;
}
}
return 0;
}
/**
* Runs all tests.
*/
int
main (int argc, char **argv)
{
const char *this_fn_name = P4EST_STRING "_test_subcomm";
/* options */
const p4est_locidx_t min_quadrants = 15;
const int min_level = 4;
const int fill_uniform = 0;
/* parallel environment */
sc_MPI_Comm mpicomm = sc_MPI_COMM_WORLD;
int mpisize, submpisize;
int mpiret;
#ifdef P4EST_ENABLE_DEBUG
int rank;
#endif
/* p4est */
p4est_connectivity_t *connectivity;
p4est_t *p4est;
p4est_locidx_t *partition;
/* initialize MPI */
mpiret = sc_MPI_Init (&argc, &argv);
SC_CHECK_MPI (mpiret);
/* exit if MPI communicator cannot be reduced */
mpiret = sc_MPI_Comm_size (mpicomm, &mpisize);
SC_CHECK_MPI (mpiret);
if (mpisize == 1) {
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
/* initialize p4est */
sc_init (mpicomm, 1, 1, NULL, SC_LP_DEFAULT);
p4est_init (NULL, SC_LP_DEFAULT);
/* create connectivity */
#ifdef P4_TO_P8
connectivity = p8est_connectivity_new_unitcube ();
#else
connectivity = p4est_connectivity_new_unitsquare ();
#endif
/* create p4est object */
p4est = p4est_new_ext (mpicomm, connectivity,
min_quadrants, min_level, fill_uniform,
0, NULL, NULL);
/* write vtk: new */
p4est_vtk_write_file (p4est, NULL, P4EST_STRING "_subcomm_new");
/* set variables pertaining to the parallel environment */
#ifdef P4EST_ENABLE_DEBUG
rank = p4est->mpirank;
#endif
submpisize = mpisize / 2;
P4EST_ASSERT (submpisize <= p4est->global_num_quadrants);
/* construct partitioning with empty ranks */
{
p4est_locidx_t n_quads_per_proc, n_quads_leftover;
int p;
partition = P4EST_ALLOC (p4est_locidx_t, mpisize);
n_quads_per_proc = p4est->global_num_quadrants / submpisize;
n_quads_leftover = p4est->global_num_quadrants -
(n_quads_per_proc * submpisize);
for (p = 0; p < mpisize; p++) {
if (p % 2) { /* if this rank will get quadrants */
partition[p] = n_quads_per_proc;
}
else { /* if this rank will be empty */
partition[p] = 0;
}
}
partition[1] += n_quads_leftover;
/* check partitioning */
#ifdef P4EST_ENABLE_DEBUG
{
p4est_gloidx_t sum = 0;
for (p = 0; p < mpisize; p++) {
sum += (p4est_gloidx_t) partition[p];
}
P4EST_ASSERT (sum == p4est->global_num_quadrants);
}
#endif
}
/*
* Test 1: Reduce MPI communicator to non-empty ranks
*/
P4EST_GLOBAL_INFOF ("%s: Into test 1\n", this_fn_name);
{
p4est_t *p4est_subcomm;
int is_nonempty;
/* create p4est copy and re-partition */
p4est_subcomm = p4est_copy_ext (p4est, 1, 1);
(void) p4est_partition_given (p4est_subcomm, partition);
/* write vtk: partitioned */
p4est_vtk_write_file (p4est_subcomm, NULL, P4EST_STRING "_subcomm_part");
/* reduce MPI communicator to non-empty ranks */
is_nonempty = p4est_comm_parallel_env_reduce (&p4est_subcomm);
P4EST_ASSERT ((is_nonempty && 0 < partition[rank]) ||
(!is_nonempty && 0 == partition[rank]));
if (is_nonempty) {
/* write vtk: reduced communicator */
p4est_vtk_write_file (p4est_subcomm, NULL,
P4EST_STRING "_subcomm_sub1");
/* destroy the p4est that has a reduced MPI communicator */
p4est_destroy (p4est_subcomm);
}
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_INFOF ("%s: Done test 1\n", this_fn_name);
/*
* Test 2: Reduce MPI communicator to non-empty ranks, but now the MPI
* communicator is not owned
*/
P4EST_GLOBAL_INFOF ("%s: Into test 2\n", this_fn_name);
{
p4est_t *p4est_subcomm;
int is_nonempty;
/* create p4est copy and re-partition */
p4est_subcomm = p4est_copy_ext (p4est, 1, 0 /* don't dup. comm. */ );
(void) p4est_partition_given (p4est_subcomm, partition);
/* reduce MPI communicator to non-empty ranks */
is_nonempty = p4est_comm_parallel_env_reduce (&p4est_subcomm);
P4EST_ASSERT ((is_nonempty && 0 < partition[rank]) ||
(!is_nonempty && 0 == partition[rank]));
if (is_nonempty) {
/* destroy the p4est that has a reduced MPI communicator */
p4est_destroy (p4est_subcomm);
}
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_INFOF ("%s: Done test 2\n", this_fn_name);
/*
* Test 3: Reduce MPI communicator to non-empty ranks, but keep rank 0
*/
P4EST_GLOBAL_INFOF ("%s: Into test 3\n", this_fn_name);
{
p4est_t *p4est_subcomm;
int sub_exists;
sc_MPI_Group group, group_reserve;
int reserve_range[1][3];
/* create group of full MPI communicator */
mpiret = sc_MPI_Comm_group (mpicomm, &group);
SC_CHECK_MPI (mpiret);
/* create sub-group containing only rank 0 */
reserve_range[0][0] = 0;
reserve_range[0][1] = 0;
reserve_range[0][2] = 1;
mpiret =
sc_MPI_Group_range_incl (group, 1, reserve_range, &group_reserve);
SC_CHECK_MPI (mpiret);
/* create p4est copy and re-partition */
p4est_subcomm = p4est_copy_ext (p4est, 1, 1);
(void) p4est_partition_given (p4est_subcomm, partition);
/* reduce MPI communicator to non-empty ranks, but keep rank 0 */
sub_exists = p4est_comm_parallel_env_reduce_ext (&p4est_subcomm,
group_reserve, 1, NULL);
P4EST_ASSERT ((sub_exists && (0 < partition[rank] || rank == 0)) ||
(!sub_exists && 0 == partition[rank]));
if (sub_exists) {
/* write vtk: reduced communicator */
p4est_vtk_write_file (p4est_subcomm, NULL,
P4EST_STRING "_subcomm_sub3");
/* destroy the p4est that has a reduced MPI communicator */
p4est_destroy (p4est_subcomm);
}
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_INFOF ("%s: Done test 3\n", this_fn_name);
/*
* Test 4: Reduce MPI communicator to non-empty ranks, but keep last 2 ranks
*/
P4EST_GLOBAL_INFOF ("%s: Into test 4\n", this_fn_name);
{
p4est_t *p4est_subcomm;
int sub_exists;
sc_MPI_Group group, group_reserve;
int reserve_range[1][3];
/* create group of full MPI communicator */
mpiret = sc_MPI_Comm_group (mpicomm, &group);
SC_CHECK_MPI (mpiret);
/* create sub-group containing only last 2 ranks */
reserve_range[0][0] = SC_MAX (0, mpisize - 2);
reserve_range[0][1] = mpisize - 1;
reserve_range[0][2] = 1;
mpiret =
sc_MPI_Group_range_incl (group, 1, reserve_range, &group_reserve);
SC_CHECK_MPI (mpiret);
/* create p4est copy and re-partition */
p4est_subcomm = p4est_copy_ext (p4est, 1, 1);
(void) p4est_partition_given (p4est_subcomm, partition);
/* reduce MPI communicator to non-empty ranks, but keep last 2 ranks */
sub_exists = p4est_comm_parallel_env_reduce_ext (&p4est_subcomm,
group_reserve, 0, NULL);
P4EST_ASSERT ((sub_exists && (0 < partition[rank] || mpisize - 2 <= rank))
|| (!sub_exists && 0 == partition[rank]));
if (sub_exists) {
/* write vtk: reduced communicator */
p4est_vtk_write_file (p4est_subcomm, NULL,
P4EST_STRING "_subcomm_sub4");
/* destroy the p4est that has a reduced MPI communicator */
p4est_destroy (p4est_subcomm);
}
}
mpiret = sc_MPI_Barrier (mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_GLOBAL_INFOF ("%s: Done test 4\n", this_fn_name);
/* destroy */
P4EST_FREE (partition);
p4est_destroy (p4est);
p4est_connectivity_destroy (connectivity);
/* finalize */
sc_finalize ();
mpiret = sc_MPI_Finalize ();
SC_CHECK_MPI (mpiret);
return 0;
}
int
p6est_comm_parallel_env_reduce_ext (p6est_t ** p6est_supercomm,
sc_MPI_Group group_add,
int add_to_beginning, int **ranks_subcomm)
{
p6est_t *p6est = *p6est_supercomm;
int mpisize = p6est->mpisize;
int mpiret;
p4est_gloidx_t *global_first_layer = p6est->global_first_layer;
p4est_gloidx_t *n_quadrants;
int submpisize;
sc_MPI_Comm submpicomm;
int *ranks;
int i;
int is_nonempty;
/* reduce MPI communicator of column layout */
is_nonempty =
p4est_comm_parallel_env_reduce_ext (&(p6est->columns), group_add,
add_to_beginning, &ranks);
/* destroy p4est and exit if this rank is empty */
if (!is_nonempty) {
p6est->columns = NULL;
p6est_destroy (p6est);
*p6est_supercomm = NULL;
if (ranks_subcomm) {
*ranks_subcomm = NULL;
}
P4EST_ASSERT (ranks == NULL);
return 0;
}
/* get sub-communicator */
submpicomm = p6est->columns->mpicomm;
/* update size of new MPI communicator */
mpiret = sc_MPI_Comm_size (submpicomm, &submpisize);
SC_CHECK_MPI (mpiret);
if (submpisize == p6est->mpisize) {
P4EST_ASSERT (ranks == NULL);
return 1;
}
/* set new parallel environment */
p6est_comm_parallel_env_release (p6est);
p6est_comm_parallel_env_assign (p6est, submpicomm);
if (p6est->columns->mpicomm_owned) {
p6est->columns->mpicomm_owned = 0;
p6est->mpicomm_owned = 1;
}
P4EST_ASSERT (p6est->mpisize == submpisize);
/* create array of non-empty processes that will be included to sub-comm */
n_quadrants = P4EST_ALLOC (p4est_gloidx_t, mpisize);
for (i = 0; i < mpisize; i++) {
n_quadrants[i] = global_first_layer[i + 1] - global_first_layer[i];
}
/* allocate and set global layer count */
P4EST_FREE (p6est->global_first_layer);
p6est->global_first_layer = P4EST_ALLOC (p4est_gloidx_t, submpisize + 1);
p6est->global_first_layer[0] = 0;
for (i = 0; i < submpisize; i++) {
P4EST_ASSERT (ranks[i] != sc_MPI_UNDEFINED);
P4EST_ASSERT (group_add != sc_MPI_GROUP_NULL
|| 0 < n_quadrants[ranks[i]]);
p6est->global_first_layer[i + 1] =
p6est->global_first_layer[i] + n_quadrants[ranks[i]];
}
P4EST_FREE (n_quadrants);
if (ranks_subcomm) {
*ranks_subcomm = ranks;
}
else {
P4EST_FREE (ranks);
}
/* return that p6est exists on this rank */
return 1;
}
trilinear_mesh_t *
p8est_trilinear_mesh_new_from_nodes (p4est_t * p4est, p4est_nodes_t * nodes)
{
const int num_procs = p4est->mpisize;
const int rank = p4est->mpirank;
int mpiret;
int k, owner;
#ifdef P4EST_DEBUG
int prev_owner = 0;
int64_t prev_fvnid = -1;
#endif
int *sharers;
size_t current, zz, num_sharers;
int32_t e, n, local_owned_end;
int64_t global_borrowed, global_shared;
int64_t local_counts[5], global_counts[5];
int32link_t *lynk, **tail;
p4est_topidx_t which_tree;
p4est_locidx_t *local_node, *shared_offsets;
p4est_tree_t *tree;
p4est_quadrant_t *q;
p4est_indep_t *in;
p8est_hang4_t *fh;
p8est_hang2_t *eh;
trilinear_elem_t *elem;
trilinear_anode_t *anode;
trilinear_dnode_t *dnode;
trilinear_mesh_t *mesh;
trilinear_mesh_pid_t *elem_pids;
trilinear_mesh_pid_t *node_pids;
sc_recycle_array_t *rarr;
P4EST_GLOBAL_PRODUCTIONF
("Into trilinear_mesh_extract with %lld total elements\n",
(long long) p4est->global_num_quadrants);
/* Allocate output data structure. */
mesh = P4EST_ALLOC_ZERO (trilinear_mesh_t, 1);
memset (mesh, -1, sizeof (*mesh));
shared_offsets = nodes->shared_offsets;
/* Assign local counts. */
P4EST_ASSERT (nodes->num_local_quadrants == p4est->local_num_quadrants);
mesh->local_elem_num = p4est->local_num_quadrants;
mesh->local_anode_num = nodes->indep_nodes.elem_count;
mesh->local_dnode_num =
nodes->face_hangings.elem_count + nodes->edge_hangings.elem_count;
mesh->local_onode_num = nodes->num_owned_indeps;
mesh->local_owned_offset = nodes->offset_owned_indeps;
mesh->local_node_num = mesh->local_anode_num + mesh->local_dnode_num;
local_owned_end = mesh->local_owned_offset + mesh->local_onode_num;
/* Communicate global counts. */
local_counts[0] = mesh->local_elem_num;
local_counts[1] = mesh->local_anode_num;
local_counts[2] = mesh->local_onode_num;
local_counts[3] = mesh->local_dnode_num;
local_counts[4] = nodes->num_owned_shared;
mpiret = MPI_Allreduce (local_counts, global_counts, 5, MPI_LONG_LONG_INT,
MPI_SUM, p4est->mpicomm);
SC_CHECK_MPI (mpiret);
P4EST_ASSERT (global_counts[0] == p4est->global_num_quadrants);
mesh->total_elem_num = global_counts[0];
global_borrowed = global_counts[1] - global_counts[2];
mesh->total_anode_num = global_counts[2];
mesh->total_dnode_num = global_counts[3];
global_shared = global_counts[4];
mesh->total_node_num = mesh->total_anode_num + mesh->total_dnode_num;
/* Allocate the mesh memory. */
mesh->elem_table = P4EST_ALLOC (trilinear_elem_t, mesh->local_elem_num);
mesh->node_table = P4EST_ALLOC (trilinear_node_t, mesh->local_node_num);
mesh->fvnid_count_table = P4EST_ALLOC (int64_t, num_procs + 1);
mesh->fvnid_interval_table = P4EST_ALLOC (int64_t, num_procs + 1);
mesh->all_fvnid_start = mesh->fvnid_interval_table;
mesh->sharer_pool = sc_mempool_new (sizeof (int32link_t));
mesh->elem_pids = P4EST_ALLOC (trilinear_mesh_pid_t, mesh->local_node_num);
mesh->node_pids = P4EST_ALLOC (trilinear_mesh_pid_t, mesh->local_node_num);
/* Assign global free variable information. */
mesh->fvnid_interval_table[0] = 0;
for (k = 0; k < num_procs; ++k) {
mesh->fvnid_interval_table[k + 1] = mesh->fvnid_interval_table[k] +
(mesh->fvnid_count_table[k] = nodes->global_owned_indeps[k]);
}
mesh->fvnid_count_table[num_procs] = -1;
mesh->global_fvnid_num = mesh->fvnid_interval_table[num_procs];
mesh->global_fvnid_start = 0;
mesh->global_fvnid_end = mesh->global_fvnid_num - 1;
P4EST_ASSERT (mesh->global_fvnid_num == mesh->total_anode_num);
/* Assign element information. */
local_node = nodes->local_nodes;
which_tree = p4est->first_local_tree;
elem_pids = mesh->elem_pids;
if (which_tree >= 0) {
tree = p4est_tree_array_index (p4est->trees, which_tree);
current = 0;
for (e = 0; e < mesh->local_elem_num; ++e) {
if (current == tree->quadrants.elem_count) {
++which_tree;
tree = p4est_tree_array_index (p4est->trees, which_tree);
current = 0;
}
q = p4est_quadrant_array_index (&tree->quadrants, current);
elem = mesh->elem_table + e;
for (k = 0; k < P4EST_CHILDREN; ++k) {
elem->local_node_id[k] = *local_node++;
}
elem->lx = (tick_t) q->x;
elem->ly = (tick_t) q->y;
elem->lz = (tick_t) q->z;
elem->size = P4EST_QUADRANT_LEN (q->level);
elem->data = q->p.user_data;
elem_pids[e] = (trilinear_mesh_pid_t) which_tree;
++current;
}
P4EST_ASSERT (which_tree == p4est->last_local_tree);
P4EST_ASSERT (current == tree->quadrants.elem_count);
}
/* Assign anchored node information. */
mesh->anode_table = mesh->node_table;
mesh->onode_table = mesh->node_table + mesh->local_owned_offset;
mesh->dnode_table = mesh->node_table + mesh->local_anode_num;
node_pids = mesh->node_pids;
for (n = 0; n < mesh->local_anode_num; ++n) {
anode = &mesh->node_table[n].anchored;
in = (p4est_indep_t *) sc_array_index (&nodes->indep_nodes, (size_t) n);
anode->point.x = in->x;
anode->point.y = in->y;
anode->point.z = in->z;
node_pids[n] = (trilinear_mesh_pid_t) in->p.piggy3.which_tree;
if (n < mesh->local_owned_offset) {
owner = nodes->nonlocal_ranks[n];
P4EST_ASSERT (owner < rank && owner >= prev_owner);
}
else if (n >= local_owned_end) {
owner = nodes->nonlocal_ranks[n - mesh->local_onode_num];
P4EST_ASSERT (owner > rank && owner >= prev_owner);
}
else {
owner = rank;
}
anode->fvnid = mesh->all_fvnid_start[owner] + in->p.piggy3.local_num;
P4EST_ASSERT (anode->fvnid > prev_fvnid);
if (in->pad8 == 0) {
P4EST_ASSERT (in->pad16 == -1);
P4EST_ASSERT (shared_offsets == NULL || shared_offsets[n] == -1);
anode->share = NULL;
}
else {
P4EST_ASSERT (in->pad8 > 0);
num_sharers = (size_t) in->pad8;
rarr =
(sc_recycle_array_t *) sc_array_index (&nodes->shared_indeps,
num_sharers - 1);
if (nodes->shared_offsets == NULL) {
P4EST_ASSERT (in->pad16 >= 0);
zz = (size_t) in->pad16;
}
else {
P4EST_ASSERT (in->pad16 == -1);
zz = (size_t) shared_offsets[n];
}
sharers = (int *) sc_array_index (&rarr->a, zz);
tail = &anode->share;
for (zz = 0; zz < num_sharers; ++zz) {
*tail = lynk = (int32link_t *) sc_mempool_alloc (mesh->sharer_pool);
lynk->id = (int32_t) sharers[zz];
tail = &lynk->next;
}
*tail = NULL;
}
#ifdef P4EST_DEBUG
prev_owner = owner;
prev_fvnid = anode->fvnid;
#endif
}
/* Assign face hanging node information. */
for (zz = 0; zz < nodes->face_hangings.elem_count; ++n, ++zz) {
dnode = &mesh->node_table[n].dangling;
fh = (p8est_hang4_t *) sc_array_index (&nodes->face_hangings, zz);
dnode->point.x = fh->x;
dnode->point.y = fh->y;
dnode->point.z = fh->z;
dnode->type = 0; /* Not used in Rhea. */
dnode->local_anode_id[0] = fh->p.piggy.depends[0];
dnode->local_anode_id[1] = fh->p.piggy.depends[1];
dnode->local_anode_id[2] = fh->p.piggy.depends[2];
dnode->local_anode_id[3] = fh->p.piggy.depends[3];
node_pids[n] = (trilinear_mesh_pid_t) fh->p.piggy.which_tree;
}
/* Assign edge hanging node information. */
for (zz = 0; zz < nodes->edge_hangings.elem_count; ++n, ++zz) {
dnode = &mesh->node_table[n].dangling;
eh = (p8est_hang2_t *) sc_array_index (&nodes->edge_hangings, zz);
dnode->point.x = eh->x;
dnode->point.y = eh->y;
dnode->point.z = eh->z;
dnode->type = 0; /* Not used in Rhea. */
dnode->local_anode_id[0] = eh->p.piggy.depends[0];
dnode->local_anode_id[1] = eh->p.piggy.depends[1];
dnode->local_anode_id[2] = dnode->local_anode_id[3] = -1;
node_pids[n] = (trilinear_mesh_pid_t) eh->p.piggy.which_tree;
}
P4EST_ASSERT (n == mesh->local_node_num);
/* Assign the remaining variables. */
mesh->mpicomm = p4est->mpicomm;
mesh->mpisize = (int32_t) num_procs;
mesh->mpirank = (int32_t) rank;
mesh->recsize = (int32_t) p4est->data_size;
mesh->destructor = p8est_trilinear_mesh_destroy;
/* These members are incomplete and need to be filled later. */
memset (mesh->bounds, 0, 6 * sizeof (int));
memset (mesh->sizes, 0, 3 * sizeof (int));
mesh->minsize = mesh->maxsize = 0;
mesh->ticksize = 0.;
mesh->extra_info = NULL;
mesh->gid = -1;
/* We are done */
P4EST_GLOBAL_PRODUCTIONF ("Done trilinear_mesh_extract"
" with %lld anodes %lld %lld\n",
(long long) mesh->total_anode_num,
(long long) global_borrowed,
(long long) global_shared);
return mesh;
}
p4est_t *
p4est_new_points (sc_MPI_Comm mpicomm, p4est_connectivity_t * connectivity,
int maxlevel, p4est_quadrant_t * points,
p4est_locidx_t num_points, p4est_locidx_t max_points,
size_t data_size, p4est_init_t init_fn, void *user_pointer)
{
int mpiret;
int num_procs, rank;
int i, isizet;
size_t lcount;
size_t *nmemb;
#ifdef P4EST_ENABLE_DEBUG
size_t zz;
#endif
p4est_topidx_t jt, num_trees;
p4est_topidx_t first_tree, last_tree, next_tree;
p4est_quadrant_t *first_quad, *next_quad, *quad;
p4est_quadrant_t a, b, c, f, l, n;
p4est_tree_t *tree;
p4est_t *p4est;
p4est_points_state_t ppstate;
P4EST_GLOBAL_PRODUCTIONF ("Into " P4EST_STRING
"_new_points with max level %d max points %lld\n",
maxlevel, (long long) max_points);
p4est_log_indent_push ();
P4EST_ASSERT (p4est_connectivity_is_valid (connectivity));
P4EST_ASSERT (max_points >= -1);
/* retrieve MPI information */
mpiret = sc_MPI_Comm_size (mpicomm, &num_procs);
SC_CHECK_MPI (mpiret);
mpiret = sc_MPI_Comm_rank (mpicomm, &rank);
SC_CHECK_MPI (mpiret);
/* This implementation runs in O(P/p * maxlevel)
* with P the total number of points, p the number of processors.
* Two optimizations are possible:
* 1. At startup remove points that lead to duplicate quadrants.
* 2. Use complete_region between successive points instead of
* the call to refine. This should give O(N/p) * maxlevel
* with N the total number of quadrants.
*/
/* parallel sort the incoming points */
lcount = (size_t) num_points;
nmemb = P4EST_ALLOC_ZERO (size_t, num_procs);
isizet = (int) sizeof (size_t);
mpiret = sc_MPI_Allgather (&lcount, isizet, sc_MPI_BYTE,
nmemb, isizet, sc_MPI_BYTE, mpicomm);
SC_CHECK_MPI (mpiret);
sc_psort (mpicomm, points, nmemb, sizeof (p4est_quadrant_t),
p4est_quadrant_compare_piggy);
P4EST_FREE (nmemb);
#ifdef P4EST_ENABLE_DEBUG
first_quad = points;
for (zz = 1; zz < lcount; ++zz) {
next_quad = points + zz;
P4EST_ASSERT (p4est_quadrant_compare_piggy (first_quad, next_quad) <= 0);
first_quad = next_quad;
}
#endif
/* create the p4est */
p4est = P4EST_ALLOC_ZERO (p4est_t, 1);
ppstate.points = points;
ppstate.num_points = num_points;
ppstate.max_points = max_points;
ppstate.current = 0;
ppstate.maxlevel = maxlevel;
/* assign some data members */
p4est->data_size = 2 * sizeof (p4est_locidx_t); /* temporary */
p4est->user_pointer = &ppstate;
p4est->connectivity = connectivity;
num_trees = connectivity->num_trees;
/* create parallel environment */
p4est_comm_parallel_env_create (p4est, mpicomm);
/* allocate memory pools */
p4est->user_data_pool = sc_mempool_new (p4est->data_size);
p4est->quadrant_pool = sc_mempool_new (sizeof (p4est_quadrant_t));
P4EST_GLOBAL_PRODUCTIONF ("New " P4EST_STRING
" with %lld trees on %d processors\n",
(long long) num_trees, num_procs);
/* allocate trees */
p4est->trees = sc_array_new (sizeof (p4est_tree_t));
sc_array_resize (p4est->trees, num_trees);
for (jt = 0; jt < num_trees; ++jt) {
tree = p4est_tree_array_index (p4est->trees, jt);
sc_array_init (&tree->quadrants, sizeof (p4est_quadrant_t));
P4EST_QUADRANT_INIT (&tree->first_desc);
P4EST_QUADRANT_INIT (&tree->last_desc);
tree->quadrants_offset = 0;
for (i = 0; i <= P4EST_QMAXLEVEL; ++i) {
tree->quadrants_per_level[i] = 0;
}
for (; i <= P4EST_MAXLEVEL; ++i) {
tree->quadrants_per_level[i] = -1;
}
tree->maxlevel = 0;
}
p4est->local_num_quadrants = 0;
p4est->global_num_quadrants = 0;
/* create point based partition */
P4EST_QUADRANT_INIT (&f);
p4est->global_first_position =
P4EST_ALLOC_ZERO (p4est_quadrant_t, num_procs + 1);
if (num_points == 0) {
P4EST_VERBOSE ("Empty processor");
first_tree = p4est->first_local_tree = -1;
first_quad = NULL;
}
else {
/* we are probably not empty */
if (rank == 0) {
first_tree = p4est->first_local_tree = 0;
p4est_quadrant_set_morton (&f, maxlevel, 0);
}
else {
first_tree = p4est->first_local_tree = points->p.which_tree;
p4est_node_to_quadrant (points, maxlevel, &f);
}
first_quad = &f;
}
last_tree = p4est->last_local_tree = -2;
p4est_comm_global_partition (p4est, first_quad);
first_quad = p4est->global_first_position + rank;
next_quad = p4est->global_first_position + (rank + 1);
next_tree = next_quad->p.which_tree;
if (first_tree >= 0 &&
p4est_quadrant_is_equal (first_quad, next_quad) &&
first_quad->p.which_tree == next_quad->p.which_tree) {
/* if all our points are consumed by the next processor we are empty */
first_tree = p4est->first_local_tree = -1;
}
if (first_tree >= 0) {
/* we are definitely not empty */
if (next_quad->x == 0 && next_quad->y == 0
#ifdef P4_TO_P8
&& next_quad->z == 0
#endif
) {
last_tree = p4est->last_local_tree = next_tree - 1;
}
else {
last_tree = p4est->last_local_tree = next_tree;
}
P4EST_ASSERT (first_tree <= last_tree);
}
/* fill the local trees */
P4EST_QUADRANT_INIT (&a);
P4EST_QUADRANT_INIT (&b);
P4EST_QUADRANT_INIT (&c);
P4EST_QUADRANT_INIT (&l);
n = *next_quad;
n.level = (int8_t) maxlevel;
for (jt = first_tree; jt <= last_tree; ++jt) {
int onlyone = 0;
int includeb = 0;
tree = p4est_tree_array_index (p4est->trees, jt);
/* determine first local quadrant of this tree */
if (jt == first_tree) {
a = *first_quad;
a.level = (int8_t) maxlevel;
first_quad = next_quad = NULL; /* free to use further down */
P4EST_ASSERT (p4est_quadrant_is_valid (&a));
}
else {
p4est_quadrant_set_morton (&a, maxlevel, 0);
P4EST_ASSERT (jt < next_tree || p4est_quadrant_compare (&a, &n) < 0);
}
/* enlarge first local quadrant if possible */
if (jt < next_tree) {
while (p4est_quadrant_child_id (&a) == 0 && a.level > 0) {
p4est_quadrant_parent (&a, &a);
}
P4EST_ASSERT (jt == first_tree || a.level == 0);
}
else {
for (c = a; p4est_quadrant_child_id (&c) == 0; a = c) {
p4est_quadrant_parent (&c, &c);
p4est_quadrant_last_descendant (&c, &l, maxlevel);
if (p4est_quadrant_compare (&l, &n) >= 0) {
break;
}
}
P4EST_ASSERT (a.level > 0);
P4EST_ASSERT ((p4est_quadrant_last_descendant (&a, &l, maxlevel),
p4est_quadrant_compare (&l, &n) < 0));
}
p4est_quadrant_first_descendant (&a, &tree->first_desc, P4EST_QMAXLEVEL);
/* determine largest possible last quadrant of this tree */
if (jt < next_tree) {
p4est_quadrant_last_descendant (&a, &l, maxlevel);
p4est_quadrant_set_morton (&b, 0, 0);
p4est_quadrant_last_descendant (&b, &b, maxlevel);
if (p4est_quadrant_is_equal (&l, &b)) {
onlyone = 1;
}
else {
includeb = 1;
for (c = b; p4est_quadrant_child_id (&c) == P4EST_CHILDREN - 1; b = c) {
p4est_quadrant_parent (&c, &c);
p4est_quadrant_first_descendant (&c, &f, maxlevel);
if (p4est_quadrant_compare (&l, &f) >= 0) {
break;
}
}
}
}
else {
b = n;
}
/* create a complete tree */
if (onlyone) {
quad = p4est_quadrant_array_push (&tree->quadrants);
*quad = a;
p4est_quadrant_init_data (p4est, jt, quad, p4est_points_init);
tree->maxlevel = a.level;
++tree->quadrants_per_level[a.level];
}
else {
p4est_complete_region (p4est, &a, 1, &b, includeb,
tree, jt, p4est_points_init);
quad = p4est_quadrant_array_index (&tree->quadrants,
tree->quadrants.elem_count - 1);
}
tree->quadrants_offset = p4est->local_num_quadrants;
p4est->local_num_quadrants += tree->quadrants.elem_count;
p4est_quadrant_last_descendant (quad, &tree->last_desc, P4EST_QMAXLEVEL);
/* verification */
#ifdef P4EST_ENABLE_DEBUG
first_quad = p4est_quadrant_array_index (&tree->quadrants, 0);
for (zz = 1; zz < tree->quadrants.elem_count; ++zz) {
next_quad = p4est_quadrant_array_index (&tree->quadrants, zz);
P4EST_ASSERT (((p4est_locidx_t *) first_quad->p.user_data)[1] ==
((p4est_locidx_t *) next_quad->p.user_data)[0]);
first_quad = next_quad;
}
#endif
}
if (last_tree >= 0) {
for (; jt < num_trees; ++jt) {
tree = p4est_tree_array_index (p4est->trees, jt);
tree->quadrants_offset = p4est->local_num_quadrants;
}
}
/* compute some member variables */
p4est->global_first_quadrant = P4EST_ALLOC (p4est_gloidx_t, num_procs + 1);
p4est_comm_count_quadrants (p4est);
/* print more statistics */
P4EST_VERBOSEF ("total local quadrants %lld\n",
(long long) p4est->local_num_quadrants);
P4EST_ASSERT (p4est_is_valid (p4est));
p4est_log_indent_pop ();
P4EST_GLOBAL_PRODUCTIONF ("Done " P4EST_STRING
"_new_points with %lld total quadrants\n",
(long long) p4est->global_num_quadrants);
/* refine to have one point per quadrant */
if (max_points >= 0) {
p4est_refine_ext (p4est, 1, maxlevel, p4est_points_refine,
p4est_points_init, NULL);
#ifdef P4EST_ENABLE_DEBUG
for (jt = first_tree; jt <= last_tree; ++jt) {
tree = p4est_tree_array_index (p4est->trees, jt);
first_quad = p4est_quadrant_array_index (&tree->quadrants, 0);
for (zz = 1; zz < tree->quadrants.elem_count; ++zz) {
next_quad = p4est_quadrant_array_index (&tree->quadrants, zz);
P4EST_ASSERT (((p4est_locidx_t *) first_quad->p.user_data)[1] ==
((p4est_locidx_t *) next_quad->p.user_data)[0]);
first_quad = next_quad;
}
}
#endif
}
/* initialize user pointer and data size */
p4est_reset_data (p4est, data_size, init_fn, user_pointer);
return p4est;
}
p4est_mesh_t *
p4est_mesh_new_ext (p4est_t * p4est, p4est_ghost_t * ghost,
int compute_tree_index, int compute_level_lists,
p4est_connect_type_t btype)
{
int do_corner = 0;
int do_volume = 0;
int rank;
p4est_locidx_t lq, ng;
p4est_locidx_t jl;
p4est_mesh_t *mesh;
P4EST_ASSERT (p4est_is_balanced (p4est, btype));
mesh = P4EST_ALLOC_ZERO (p4est_mesh_t, 1);
lq = mesh->local_num_quadrants = p4est->local_num_quadrants;
ng = mesh->ghost_num_quadrants = (p4est_locidx_t) ghost->ghosts.elem_count;
if (btype == P4EST_CONNECT_FULL) {
do_corner = 1;
}
do_volume = (compute_tree_index || compute_level_lists ? 1 : 0);
/* Optional map of tree index for each quadrant */
if (compute_tree_index) {
mesh->quad_to_tree = P4EST_ALLOC (p4est_topidx_t, lq);
}
mesh->ghost_to_proc = P4EST_ALLOC (int, ng);
mesh->quad_to_quad = P4EST_ALLOC (p4est_locidx_t, P4EST_FACES * lq);
mesh->quad_to_face = P4EST_ALLOC (int8_t, P4EST_FACES * lq);
mesh->quad_to_half = sc_array_new (P4EST_HALF * sizeof (p4est_locidx_t));
/* Optional per-level lists of quadrants */
if (compute_level_lists) {
mesh->quad_level = P4EST_ALLOC (sc_array_t, P4EST_QMAXLEVEL + 1);
for (jl = 0; jl <= P4EST_QMAXLEVEL; ++jl) {
sc_array_init (mesh->quad_level + jl, sizeof (p4est_locidx_t));
}
}
/* Populate ghost information */
rank = 0;
for (jl = 0; jl < ng; ++jl) {
while (ghost->proc_offsets[rank + 1] <= jl) {
++rank;
P4EST_ASSERT (rank < p4est->mpisize);
}
mesh->ghost_to_proc[jl] = rank;
}
/* Fill face arrays with default values */
memset (mesh->quad_to_quad, -1, P4EST_FACES * lq * sizeof (p4est_locidx_t));
memset (mesh->quad_to_face, -25, P4EST_FACES * lq * sizeof (int8_t));
if (do_corner) {
/* Initialize corner information to a consistent state */
mesh->quad_to_corner = P4EST_ALLOC (p4est_locidx_t, P4EST_CHILDREN * lq);
memset (mesh->quad_to_corner, -1, P4EST_CHILDREN * lq *
sizeof (p4est_locidx_t));
mesh->corner_offset = sc_array_new (sizeof (p4est_locidx_t));
*(p4est_locidx_t *) sc_array_push (mesh->corner_offset) = 0;
mesh->corner_quad = sc_array_new (sizeof (p4est_locidx_t));
mesh->corner_corner = sc_array_new (sizeof (int8_t));
}
/* Call the forest iterator to collect face connectivity */
p4est_iterate (p4est, ghost, mesh,
(do_volume ? mesh_iter_volume : NULL), mesh_iter_face,
#ifdef P4_TO_P8
NULL,
#endif
do_corner ? mesh_iter_corner : NULL);
return mesh;
}
