void exchanger_verify(exchanger_t* ex, void (*handler)(const char* format, ...))
{
#if POLYMEC_HAVE_MPI
START_FUNCTION_TIMER();
// An exchanger is valid/consistent iff the number of elements that
// are exchanged between any two processors are agreed upon between those
// two processors. So we send our expected number of exchanged elements
// to our neighbors, and receive their numbers, and then compare.
int num_send_procs = ex->send_map->size;
int num_receive_procs = ex->receive_map->size;
int num_sent_elements[num_send_procs],
num_elements_expected_by_neighbors[num_send_procs],
send_procs[num_send_procs];
MPI_Request requests[num_send_procs + num_receive_procs];
int pos = 0, proc, *indices, num_indices;
// Post receives for numbers of elements we send to others.
int p = 0;
while (exchanger_next_send(ex, &pos, &proc, &indices, &num_indices))
{
num_sent_elements[p] = num_indices;
send_procs[p] = proc;
int err = MPI_Irecv(&num_elements_expected_by_neighbors[p], 1,
MPI_INT, proc, 0, ex->comm, &requests[p]);
if (err != MPI_SUCCESS)
polymec_error("exchanger_verify: Could not receive sent element data from %d", proc);
++p;
}
// Send our receive neighbors the number that we expect to receive. This
// means that we will get the number that our neighbors expect to receive,
// which should be the same number as what we're sending.
pos = p = 0;
while (exchanger_next_receive(ex, &pos, &proc, &indices, &num_indices))
{
int err = MPI_Isend(&num_indices, 1, MPI_INT, proc, 0, ex->comm, &requests[num_send_procs + p]);
if (err != MPI_SUCCESS)
polymec_error("exchanger_verify: Could not send number of received elements to %d", proc);
++p;
}
// Wait for messages to fly.
MPI_Status statuses[num_send_procs + num_receive_procs];
MPI_Waitall(num_send_procs + num_receive_procs, requests, statuses);
// Now verify that the numbers are the same.
for (p = 0; p < num_send_procs; ++p)
{
if (num_sent_elements[p] != num_elements_expected_by_neighbors[p])
{
handler("exchanger_verify: Sending %d elements to proc %d, but %d elements are expected.",
num_sent_elements[p], send_procs[p],
num_elements_expected_by_neighbors[p]);
}
}
STOP_FUNCTION_TIMER();
#endif
}
void use_units_system(int units_system_name)
{
if (units_systems == NULL)
polymec_error("use_units_system: No valid units system is defined!");
int_real_unordered_map_t** ptr = (int_real_unordered_map_t**)int_ptr_unordered_map_get(units_systems, units_system_name);
if (ptr == NULL)
polymec_error("use_units_system: units system %d is not defined!", units_system_name);
current_units = *ptr;
}
void set_physical_constant(int physical_constant_name, real_t value, int units_system_name)
{
if (units_systems == NULL)
polymec_error("set_physical_constant: No valid units system is defined!");
int_real_unordered_map_t** ptr = (int_real_unordered_map_t**)int_ptr_unordered_map_get(units_systems, units_system_name);
if (ptr == NULL)
polymec_error("set_physical_constant: units system %d is not defined!", units_system_name);
int_real_unordered_map_t* units = *ptr;
int_real_unordered_map_insert(units, physical_constant_name, value);
}
static size_t mpi_size(MPI_Datatype type)
{
size_t size = 0;
if (type == MPI_REAL_T)
size = sizeof(real_t);
else if (type == MPI_DOUBLE)
size = sizeof(double);
else if (type == MPI_FLOAT)
size = sizeof(float);
else if (type == MPI_INT)
size = sizeof(int);
else if (type == MPI_LONG)
size = sizeof(long);
else if (type == MPI_LONG_LONG)
size = sizeof(long long);
else if (type == MPI_UINT64_T)
size = sizeof(uint64_t);
else if (type == MPI_CHAR)
size = sizeof(char);
else if (type == MPI_BYTE)
size = sizeof(uint8_t);
else
polymec_error("Unsupported MPI data type used.");
return size;
}
exodus_file_t* exodus_file_open(MPI_Comm comm,
const char* filename)
{
if (!file_exists(filename))
polymec_error("exodus_file_open: %s does not exist.", filename);
return open_exodus_file(comm, filename, EX_READ);
}
void define_units_system(int units_system_name)
{
if (units_systems == NULL)
units_systems = int_ptr_unordered_map_new();
if (int_ptr_unordered_map_contains(units_systems, units_system_name))
polymec_error("define_units_system: units system %d is already defined.", units_system_name);
int_ptr_unordered_map_insert_with_v_dtor(units_systems,
units_system_name,
int_real_unordered_map_new(),
DTOR(int_real_unordered_map_free));
}
void amr_grid_finalize(amr_grid_t* grid)
{
ASSERT(!grid->finalized);
// Make sure that we have accounted for all patches in our construction
// process.
DECLARE_3D_ARRAY(patch_type_t, patch_types, grid->patch_types, grid->nx, grid->ny, grid->nz);
if (grid->num_local_patches < (grid->nx * grid->ny * grid->nz))
{
for (int i = 0; i < grid->nx; ++i)
{
for (int j = 0; j < grid->ny; ++j)
{
for (int k = 0; k < grid->nz; ++k)
{
if (patch_types[i][j][k] == NO_PATCH)
polymec_error("amr_grid_finalize: no local or remote patch at (%d, %d, %d).", i, j, k);
}
}
}
}
// Make an array of indices for locally-present patches.
grid->local_patch_indices = polymec_malloc(sizeof(int) * 3 * grid->num_local_patches);
int l = 0;
for (int i = 0; i < grid->nx; ++i)
{
for (int j = 0; j < grid->ny; ++j)
{
for (int k = 0; k < grid->nz; ++k, ++l)
{
grid->local_patch_indices[3*l] = i;
grid->local_patch_indices[3*l+1] = j;
grid->local_patch_indices[3*l+2] = k;
}
}
}
// Establish our remote copying pattern.
grid->cell_ex = grid_cell_exchanger_new(grid);
grid->x_face_ex = grid_x_face_exchanger_new(grid);
grid->y_face_ex = grid_y_face_exchanger_new(grid);
grid->z_face_ex = grid_z_face_exchanger_new(grid);
grid->x_edge_ex = grid_x_edge_exchanger_new(grid);
grid->y_edge_ex = grid_y_edge_exchanger_new(grid);
grid->z_edge_ex = grid_z_edge_exchanger_new(grid);
grid->node_ex = grid_node_exchanger_new(grid);
grid->finalized = true;
}
polygon2_t* polygon2_star(point2_t* x0, point2_t* points, int num_points)
{
// Make sure x0 is not one of the points.
for (int i = 0; i < num_points; ++i)
{
if (point2_distance(x0, &points[i]) < 1e-14)
polymec_error("polygon2_star: Point %d coincides with x0.", i);
}
// Sort the points by angles.
star_angle_t angles[num_points];
for (int i = 0; i < num_points; ++i)
{
angles[i].angle = atan2(points[i].y - x0->y, points[i].x - x0->x);
angles[i].index = i;
}
qsort(angles, (size_t)num_points, sizeof(star_angle_t), star_angle_cmp);
// Create a polygon from the new ordering.
int ordering[num_points];
for (int i = 0; i < num_points; ++i)
ordering[i] = angles[i].index;
return polygon2_new_with_ordering(points, ordering, num_points);
}
mesh_t* create_uniform_mesh_on_rank(MPI_Comm comm, int rank,
int nx, int ny, int nz, bbox_t* bbox)
{
ASSERT(comm != MPI_COMM_SELF);
ASSERT(rank >= 0);
int my_rank, nprocs;
MPI_Comm_rank(comm, &my_rank);
MPI_Comm_rank(comm, &nprocs);
if (rank > nprocs)
polymec_error("create_uniform_mesh_on_rank: invalid rank: %d", rank);
mesh_t* mesh = NULL;
if (my_rank == rank)
mesh = create_uniform_mesh(comm, nx, ny, nz, bbox);
else
{
// Initialize serializers.
serializer_t* s = cubic_lattice_serializer();
s = bbox_serializer();
}
return mesh;
}
// This helper sets up the exchanger ex so that it can migrate data from the
// current process according to the local partition vector.
exchanger_t* create_migrator(MPI_Comm comm,
int64_t* local_partition,
int num_vertices)
{
START_FUNCTION_TIMER();
exchanger_t* migrator = exchanger_new(comm);
// Tally up the number of vertices we're going to send to every other process,
// including those that are staying on our own.
int nprocs, rank;
MPI_Comm_size(comm, &nprocs);
MPI_Comm_rank(comm, &rank);
int num_vertices_to_send[nprocs];
memset(num_vertices_to_send, 0, sizeof(int) * nprocs);
for (int v = 0; v < num_vertices; ++v)
num_vertices_to_send[local_partition[v]]++;
// Get the number of vertices we're going to receive from every other process.
int num_vertices_to_receive[nprocs];
MPI_Alltoall(num_vertices_to_send, 1, MPI_INT,
num_vertices_to_receive, 1, MPI_INT, comm);
// Send and receive the actual vertices.
int** receive_vertices = polymec_malloc(sizeof(int*) * nprocs);
memset(receive_vertices, 0, sizeof(int*) * nprocs);
int num_requests = 0;
for (int p = 0; p < nprocs; ++p)
{
if ((rank != p) && (num_vertices_to_receive[p] > 0)) ++num_requests;
if ((rank != p) && (num_vertices_to_send[p] > 0)) ++num_requests;
}
if (num_requests > 0)
{
MPI_Request requests[num_requests];
int r = 0;
for (int p = 0; p < nprocs; ++p)
{
if (num_vertices_to_receive[p] > 0)
{
receive_vertices[p] = polymec_malloc(sizeof(int) * num_vertices_to_receive[p]);
if (p != rank)
{
// Note that we use this rank as our tag.
MPI_Irecv(receive_vertices[p], num_vertices_to_receive[p], MPI_INT, p, rank, comm, &requests[r++]);
}
}
if (num_vertices_to_send[p] > 0)
{
int send_vertices[num_vertices_to_send[p]], s = 0;
for (int v = 0; v < num_vertices; ++v)
{
if (local_partition[v] == p)
send_vertices[s++] = v;
}
if (p != rank)
{
// Note that we use the destination rank p as our tag.
exchanger_set_send(migrator, p, send_vertices, num_vertices_to_send[p], true);
MPI_Isend(send_vertices, num_vertices_to_send[p], MPI_INT, p, p, comm, &requests[r++]);
}
else
memcpy(receive_vertices[rank], send_vertices, sizeof(int) * num_vertices_to_receive[rank]);
}
}
ASSERT(r == num_requests);
// Wait for exchanges to finish. We can't use MPI_Waitall here because
// not all processes necessary participate.
MPI_Status statuses[num_requests];
int finished[num_requests];
memset(finished, 0, num_requests*sizeof(int));
while (true)
{
bool all_finished = true;
for (int i = 0; i < num_requests; ++i)
{
if (!finished[i])
{
if (MPI_Test(&requests[i], &finished[i], &statuses[i]) != MPI_SUCCESS)
polymec_error("create_migrator: Internal error.");
if (!finished[i]) all_finished = false;
}
}
// If the transmissions have finished at this point, we
// can break out of the loop.
if (all_finished) break;
}
// Now register all the vertices we're receiving with the migrator.
for (int p = 0; p < nprocs; ++p)
{
if (num_vertices_to_receive[p] > 0)
{
ASSERT(receive_vertices[p] != NULL);
if (rank != p)
exchanger_set_receive(migrator, p, receive_vertices[p], num_vertices_to_receive[p], true);
polymec_free(receive_vertices[p]);
}
}
polymec_free(receive_vertices);
}
STOP_FUNCTION_TIMER();
return migrator;
}
void exchanger_finish_transfer(exchanger_t* ex, int token)
{
#if POLYMEC_HAVE_MPI
START_FUNCTION_TIMER();
ASSERT(token >= 0);
ASSERT(token < ex->num_pending_msgs);
ASSERT(ex->transfer_counts[token] != NULL); // We can't finish an exchange as a transfer!
// Retrieve the message for the given token.
mpi_message_t* msg = ex->pending_msgs[token];
int* count = ex->transfer_counts[token];
int err = exchanger_waitall(ex, msg);
if (err != MPI_SUCCESS) return;
// Copy the received data into the original array.
void* orig_buffer = ex->orig_buffers[token];
mpi_message_unpack(msg, orig_buffer, ex->receive_offset, ex->receive_map);
// Now cull the sent elements from the array.
int num_sent = 0, pos = 0, proc;
exchanger_channel_t* c;
while (exchanger_map_next(ex->send_map, &pos, &proc, &c))
{
// Move the sent elements to the back.
int stride = msg->stride;
if (msg->type == MPI_REAL_T)
{
real_t* array = (real_t*)orig_buffer;
for (int i = 0; i < c->num_indices; ++i)
{
for (int s = 0; s < stride; ++s)
{
array[stride*c->indices[i]+s] = array[*count-1-num_sent];
++num_sent;
}
}
}
else if (msg->type == MPI_DOUBLE)
{
double* array = (double*)orig_buffer;
for (int i = 0; i < c->num_indices; ++i)
{
for (int s = 0; s < stride; ++s)
{
array[stride*c->indices[i]+s] = array[*count-1-num_sent];
++num_sent;
}
}
}
else
{
// FIXME: What about other data types???
polymec_error("exchanger_finish_transfer: unimplemented data type.");
}
}
// Convey the change in count of the transferred data.
*count -= num_sent;
ASSERT(*count >= 0);
// Pull the message out of our list of pending messages and delete it.
ex->pending_msgs[token] = NULL;
ex->orig_buffers[token] = NULL;
ex->transfer_counts[token] = NULL;
mpi_message_free(msg);
STOP_FUNCTION_TIMER();
#endif
}
void exodus_file_write_mesh(exodus_file_t* file,
fe_mesh_t* mesh)
{
ASSERT(file->writing);
// See whether we have polyhedral blocks, and whether the non-polyhedral
// blocks have supported element types.
int num_blocks = fe_mesh_num_blocks(mesh);
bool is_polyhedral = false;
int pos = 0;
char* block_name;
fe_block_t* block;
while (fe_mesh_next_block(mesh, &pos, &block_name, &block))
{
fe_mesh_element_t elem_type = fe_block_element_type(block);
if (elem_type == FE_POLYHEDRON)
{
is_polyhedral = true;
break;
}
else if (elem_type != FE_INVALID)
{
// Check the number of nodes for the element.
if (!element_is_supported(elem_type, fe_block_num_element_nodes(block, 0)))
polymec_error("exodus_file_write_mesh: Element type in block %s has invalid number of nodes.", block_name);
}
else
polymec_error("exodus_file_write_mesh: Invalid element type for block %s.", block_name);
}
// Write out information about elements, faces, edges, nodes.
file->num_nodes = fe_mesh_num_nodes(mesh);
ex_init_params params;
strcpy(params.title, file->title);
params.num_dim = 3;
params.num_nodes = file->num_nodes;
int num_edges = fe_mesh_num_edges(mesh);
params.num_edge = num_edges;
params.num_edge_blk = 0;
int num_faces = fe_mesh_num_faces(mesh);
params.num_face = num_faces;
params.num_face_blk = (is_polyhedral) ? 1 : 0;
int num_elem = fe_mesh_num_elements(mesh);
params.num_elem = num_elem;
params.num_elem_blk = num_blocks;
params.num_elem_sets = file->num_elem_sets = fe_mesh_num_element_sets(mesh);
params.num_face_sets = file->num_face_sets = fe_mesh_num_face_sets(mesh);
params.num_edge_sets = file->num_edge_sets = fe_mesh_num_edge_sets(mesh);
params.num_node_sets = file->num_node_sets = fe_mesh_num_node_sets(mesh);
params.num_side_sets = file->num_side_sets = fe_mesh_num_side_sets(mesh);
params.num_elem_maps = 0;
params.num_face_maps = 0;
params.num_edge_maps = 0;
params.num_node_maps = 0;
ex_put_init_ext(file->ex_id, ¶ms);
// If we have any polyhedral element blocks, we write out a single face
// block that incorporates all of the polyhedral elements.
if (is_polyhedral)
{
// Generate face->node connectivity information.
int num_pfaces = fe_mesh_num_faces(mesh);
int face_node_size = 0;
int num_face_nodes[num_pfaces];
for (int f = 0; f < num_pfaces; ++f)
{
int num_nodes = fe_mesh_num_face_nodes(mesh, f);
num_face_nodes[f] = num_nodes;
face_node_size += num_nodes;
}
int* face_nodes = polymec_malloc(sizeof(int) * face_node_size);
int offset = 0;
for (int f = 0; f < num_pfaces; ++f)
{
fe_mesh_get_face_nodes(mesh, f, &face_nodes[offset]);
offset += num_face_nodes[f];
}
for (int i = 0; i < face_node_size; ++i)
face_nodes[i] += 1;
// Write an "nsided" face block.
ex_put_block(file->ex_id, EX_FACE_BLOCK, 1, "nsided",
num_pfaces, face_node_size, 0, 0, 0);
ex_put_name(file->ex_id, EX_FACE_BLOCK, 1, "face_block");
ex_put_conn(file->ex_id, EX_FACE_BLOCK, 1, face_nodes, NULL, NULL);
// Clean up.
polymec_free(face_nodes);
// Number of nodes per face.
ex_put_entity_count_per_polyhedra(file->ex_id, EX_FACE_BLOCK,
1, num_face_nodes);
}
// Go over the element blocks and write out the data.
pos = 0;
while (fe_mesh_next_block(mesh, &pos, &block_name, &block))
{
int elem_block = pos;
int num_e = fe_block_num_elements(block);
fe_mesh_element_t elem_type = fe_block_element_type(block);
if (elem_type == FE_POLYHEDRON)
{
// Count up the faces in the block and write the block information.
int tot_num_elem_faces = 0;
int faces_per_elem[num_e];
for (int i = 0; i < num_e; ++i)
{
faces_per_elem[i] = fe_block_num_element_faces(block, i);
tot_num_elem_faces += faces_per_elem[i];
}
ex_put_block(file->ex_id, EX_ELEM_BLOCK, elem_block, "nfaced",
num_e, 0, 0, tot_num_elem_faces, 0);
// Write elem->face connectivity information.
int elem_faces[tot_num_elem_faces], offset = 0;
for (int i = 0; i < num_e; ++i)
{
fe_block_get_element_faces(block, i, &elem_faces[offset]);
offset += faces_per_elem[i];
}
for (int i = 0; i < tot_num_elem_faces; ++i)
elem_faces[i] += 1;
ex_put_conn(file->ex_id, EX_ELEM_BLOCK, elem_block, NULL, NULL, elem_faces);
ex_put_entity_count_per_polyhedra(file->ex_id, EX_ELEM_BLOCK, elem_block, faces_per_elem);
}
else if (elem_type != FE_INVALID)
{
// Get element information.
char elem_type_name[MAX_NAME_LENGTH+1];
get_elem_name(elem_type, elem_type_name);
int num_nodes_per_elem = fe_block_num_element_nodes(block, 0);
// Write the block.
ex_put_block(file->ex_id, EX_ELEM_BLOCK, elem_block, elem_type_name,
num_e, num_nodes_per_elem, 0, 0, 0);
// Write the elem->node connectivity.
int elem_nodes[num_e* num_nodes_per_elem], offset = 0;
for (int i = 0; i < num_e; ++i)
{
fe_block_get_element_nodes(block, i, &elem_nodes[offset]);
offset += num_nodes_per_elem;
}
for (int i = 0; i < num_e* num_nodes_per_elem; ++i)
elem_nodes[i] += 1;
ex_put_conn(file->ex_id, EX_ELEM_BLOCK, elem_block, elem_nodes, NULL, NULL);
}
// Set the element block name.
ex_put_name(file->ex_id, EX_ELEM_BLOCK, elem_block, block_name);
}
// Set node positions.
real_t x[file->num_nodes], y[file->num_nodes], z[file->num_nodes];
point_t* X = fe_mesh_node_positions(mesh);
for (int n = 0; n < file->num_nodes; ++n)
{
x[n] = X[n].x;
y[n] = X[n].y;
z[n] = X[n].z;
}
ex_put_coord(file->ex_id, x, y, z);
char* coord_names[3] = {"x", "y", "z"};
ex_put_coord_names(file->ex_id, coord_names);
// Write sets of entities.
int *set, set_id = 0;
size_t set_size;
char* set_name;
pos = set_id = 0;
while (fe_mesh_next_element_set(mesh, &pos, &set_name, &set, &set_size))
write_set(file, EX_ELEM_SET, ++set_id, set_name, set, set_size);
pos = set_id = 0;
while (fe_mesh_next_face_set(mesh, &pos, &set_name, &set, &set_size))
write_set(file, EX_FACE_SET, ++set_id, set_name, set, set_size);
pos = set_id = 0;
while (fe_mesh_next_edge_set(mesh, &pos, &set_name, &set, &set_size))
write_set(file, EX_EDGE_SET, ++set_id, set_name, set, set_size);
pos = set_id = 0;
while (fe_mesh_next_node_set(mesh, &pos, &set_name, &set, &set_size))
write_set(file, EX_NODE_SET, ++set_id, set_name, set, set_size);
pos = set_id = 0;
while (fe_mesh_next_side_set(mesh, &pos, &set_name, &set, &set_size))
write_set(file, EX_SIDE_SET, ++set_id, set_name, set, set_size);
}
void create_boundary_generators(ptr_array_t* surface_points,
ptr_array_t* surface_normals,
ptr_array_t* surface_tags,
point_t** boundary_generators,
int* num_boundary_generators,
char*** tag_names,
int_array_t*** tags,
int* num_tags)
{
ASSERT(surface_points->size >= 4); // surface must be closed!
ASSERT(surface_points->size == surface_normals->size);
ASSERT(surface_points->size == surface_tags->size);
int num_surface_points = surface_points->size;
// Compute the minimum distance from each surface point to its neighbors.
real_t* h_min = polymec_malloc(sizeof(real_t) * num_surface_points);
{
// Dump the surface points into a kd-tree.
point_t* surf_points = polymec_malloc(sizeof(point_t) * num_surface_points);
for (int i = 0; i < num_surface_points; ++i)
surf_points[i] = *((point_t*)surface_points->data[i]);
kd_tree_t* tree = kd_tree_new(surf_points, num_surface_points);
int neighbors[2];
for (int i = 0; i < num_surface_points; ++i)
{
// Find the "nearest 2" points to the ith surface point--the first is
// the point itself, and the second is its nearest neighbor.
// FIXME: Serious memory error within here. We work around it for
// FIXME: the moment by freshing the tree pointer, which is
// FIXME: corrupted. ICK!
kd_tree_t* tree_p = tree;
kd_tree_nearest_n(tree, &surf_points[i], 2, neighbors);
tree = tree_p;
ASSERT(neighbors[0] == i);
ASSERT(neighbors[1] >= 0);
ASSERT(neighbors[1] < num_surface_points);
h_min[i] = point_distance(&surf_points[i], &surf_points[neighbors[1]]);
}
// Clean up.
kd_tree_free(tree);
polymec_free(surf_points);
}
// Generate boundary points for each surface point based on how many
// surfaces it belongs to.
ptr_array_t* boundary_points = ptr_array_new();
string_int_unordered_map_t* tag_indices = string_int_unordered_map_new();
ptr_array_t* boundary_tags = ptr_array_new();
for (int i = 0; i < num_surface_points; ++i)
{
// Add any tags from this point to the set of existing boundary tags.
string_slist_t* tags = surface_tags->data[i];
for (string_slist_node_t* t_iter = tags->front; t_iter != NULL; t_iter = t_iter->next)
string_int_unordered_map_insert(tag_indices, t_iter->value, tag_indices->size);
// Retrieve the surface point.
point_t* x_surf = surface_points->data[i];
// Retrieve the list of normal vectors for this surface point.
ptr_slist_t* normal_list = surface_normals->data[i];
int num_normals = normal_list->size;
vector_t normals[num_normals];
int n_offset = 0;
for (ptr_slist_node_t* n_iter = normal_list->front; n_iter != NULL; n_iter = n_iter->next)
normals[n_offset++] = *((vector_t*)n_iter->value);
// For now, let's keep things relatively simple.
if (num_normals > 2)
{
polymec_error("create_boundary_generators: Too many normal vectors (%d) for surface point %d at (%g, %g, %g)",
num_normals, i, x_surf->x, x_surf->y, x_surf->z);
}
// Create boundary points based on this list of normals.
if (num_normals == 1)
{
// This point only belongs to one surface, so we create boundary points
// on either side of it.
point_t* x_out = polymec_malloc(sizeof(point_t));
x_out->x = x_surf->x + h_min[i]*normals[0].x;
x_out->y = x_surf->y + h_min[i]*normals[0].y;
x_out->z = x_surf->z + h_min[i]*normals[0].z;
ptr_array_append_with_dtor(boundary_points, x_out, polymec_free);
point_t* x_in = polymec_malloc(sizeof(point_t));
x_in->x = x_surf->x - h_min[i]*normals[0].x;
x_in->y = x_surf->y - h_min[i]*normals[0].y;
x_in->z = x_surf->z - h_min[i]*normals[0].z;
ptr_array_append_with_dtor(boundary_points, x_in, polymec_free);
}
else if (num_normals == 2)
{
// This point appears at the interface between two surfaces.
// (Or so it seems.)
ASSERT(vector_dot(&normals[0], &normals[1]) < 0.0);
point_t* x1 = polymec_malloc(sizeof(point_t));
x1->x = x_surf->x + h_min[i]*normals[0].x;
x1->y = x_surf->y + h_min[i]*normals[0].y;
x1->z = x_surf->z + h_min[i]*normals[0].z;
ptr_array_append_with_dtor(boundary_points, x1, polymec_free);
point_t* x2 = polymec_malloc(sizeof(point_t));
x2->x = x_surf->x - h_min[i]*normals[1].x;
x2->y = x_surf->y - h_min[i]*normals[1].y;
x2->z = x_surf->z - h_min[i]*normals[1].z;
ptr_array_append_with_dtor(boundary_points, x2, polymec_free);
}
// Tag the boundary point appropriately.
ptr_array_append(boundary_tags, tags); // Borrowed ref to tags.
}
// Move the surface points into a contiguous array.
*boundary_generators = polymec_malloc(sizeof(point_t) * boundary_points->size);
*num_boundary_generators = boundary_points->size;
for (int i = 0; i < boundary_points->size; ++i)
{
(*boundary_generators)[i] = *((point_t*)boundary_points->data[i]);
}
// Transcribe the tags.
*tag_names = polymec_malloc(sizeof(char*) * tag_indices->size);
*tags = polymec_malloc(sizeof(int_array_t*) * tag_indices->size);
*num_tags = tag_indices->size;
char* tag_name;
int pos = 0, tag_index;
while (string_int_unordered_map_next(tag_indices, &pos, &tag_name, &tag_index))
{
(*tag_names)[tag_index] = string_dup(tag_name);
(*tags)[tag_index] = int_array_new();
for (int j = 0; j < *num_boundary_generators; ++j)
int_array_append((*tags)[tag_index], tag_index);
}
// Clean up.
string_int_unordered_map_free(tag_indices);
polymec_free(h_min);
}
// Copies data from the patch buffer to the patches in the given grid.
static void patch_buffer_copy_out(patch_buffer_t* buffer, amr_grid_data_t* grid_data)
{
amr_grid_t* grid = amr_grid_data_grid(grid_data);
amr_grid_data_centering_t centering = amr_grid_data_centering(grid_data);
int num_components = amr_grid_data_num_components(grid_data);
int num_ghosts = amr_grid_data_num_ghosts(grid_data);
// Now count up the sent data for each patch.
int local_mask = LOCAL_SAME_LEVEL | LOCAL_FINER_LEVEL | LOCAL_COARSER_LEVEL;
int pos = 0, i, j, k, p = 0;
amr_patch_t* patch;
DECLARE_3D_ARRAY(patch_type_t, patch_types, grid->patch_types, grid->nx, grid->ny, grid->nz);
while (amr_grid_data_next_local_patch(grid_data, &pos, &i, &j, &k, &patch))
{
patch_type_t patch_type = patch_types[i][j][k];
if (patch_type == LOCAL_SAME_LEVEL)
polymec_error("patch_buffer_copy_out: adaptive mesh refinement is not yet supported!");
DECLARE_AMR_PATCH_ARRAY(array, patch);
int offset = buffer->patch_receive_offsets[p];
// Copy out -x values.
int num_values = num_transmitted_patch_values(grid, centering, local_mask, i, j, k, i-1, j, k);
for (int j = patch->j1; j < patch->j2; ++j)
for (int k = patch->k1; k < patch->k2; ++k)
for (int g = 0; g < num_ghosts; ++g)
for (int c = 0; c < num_components; ++c, ++offset)
array[g][j][k][c] = buffer->data[offset];
// Copy out +x values.
num_values = num_transmitted_patch_values(grid, centering, local_mask, i, j, k, i+1, j, k);
for (int j = patch->j1; j < patch->j2; ++j)
for (int k = patch->k1; k < patch->k2; ++k)
for (int g = 0; g < num_ghosts; ++g)
for (int c = 0; c < num_components; ++c, ++offset)
array[patch->i2+num_ghosts-g-1][j][k][c] = buffer->data[offset];
// Copy out -y values.
num_values = num_transmitted_patch_values(grid, centering, local_mask, i, j, k, i, j-1, k);
for (int i = patch->i1; i < patch->i2; ++i)
for (int k = patch->k1; k < patch->k2; ++k)
for (int g = 0; g < num_ghosts; ++g)
for (int c = 0; c < num_components; ++c, ++offset)
array[i][g][k][c] = buffer->data[offset];
// Copy out +y values.
num_values = num_transmitted_patch_values(grid, centering, local_mask, i, j, k, i, j+1, k);
for (int i = patch->i1; i < patch->i2; ++i)
for (int k = patch->k1; k < patch->k2; ++k)
for (int g = 0; g < num_ghosts; ++g)
for (int c = 0; c < num_components; ++c, ++offset)
array[i][patch->j2+num_ghosts-g-1][k][c] = buffer->data[offset];
// Copy out -z values.
num_values = num_transmitted_patch_values(grid, centering, local_mask, i, j, k, i, j, k-1);
for (int i = patch->i1; i < patch->i2; ++i)
for (int j = patch->j1; j < patch->j2; ++j)
for (int g = 0; g < num_ghosts; ++g)
for (int c = 0; c < num_components; ++c, ++offset)
array[i][j][g][c] = buffer->data[offset];
// Copy out +z values.
num_values = num_transmitted_patch_values(grid, centering, local_mask, i, j, k, i, j, k+1);
for (int i = patch->i1; i < patch->i2; ++i)
for (int j = patch->j1; j < patch->j2; ++j)
for (int g = 0; g < num_ghosts; ++g)
for (int c = 0; c < num_components; ++c, ++offset)
array[i][j][patch->k2+num_ghosts-g-1][c] = buffer->data[offset];
ASSERT(offset == buffer->patch_receive_offsets[p+1]);
++p;
}
}
fe_mesh_t* exodus_file_read_mesh(exodus_file_t* file)
{
// Create the "host" FE mesh.
fe_mesh_t* mesh = fe_mesh_new(file->comm, file->num_nodes);
// Count up the number of polyhedral blocks.
int num_poly_blocks = 0;
for (int i = 0; i < file->num_elem_blocks; ++i)
{
int elem_block = file->elem_block_ids[i];
char elem_type_name[MAX_NAME_LENGTH+1];
int num_elem, num_nodes_per_elem, num_faces_per_elem;
ex_get_block(file->ex_id, EX_ELEM_BLOCK, elem_block,
elem_type_name, &num_elem,
&num_nodes_per_elem, NULL,
&num_faces_per_elem, NULL);
fe_mesh_element_t elem_type = get_element_type(elem_type_name);
if (elem_type == FE_POLYHEDRON)
++num_poly_blocks;
}
// If we have any polyhedral element blocks, we read a single face
// block that incorporates all of the polyhedral elements.
if (num_poly_blocks > 0)
{
// Dig up the face block corresponding to this element block.
char face_type[MAX_NAME_LENGTH+1];
int num_faces, num_nodes;
ex_get_block(file->ex_id, EX_FACE_BLOCK, file->face_block_ids[0], face_type, &num_faces,
&num_nodes, NULL, NULL, NULL);
if (string_ncasecmp(face_type, "nsided", 6) != 0)
{
fe_mesh_free(mesh);
ex_close(file->ex_id);
polymec_error("Invalid face type for polyhedral element block.");
}
// Find the number of nodes for each face in the block.
int* num_face_nodes = polymec_malloc(sizeof(int) * num_faces);
ex_get_entity_count_per_polyhedra(file->ex_id, EX_FACE_BLOCK,
file->face_block_ids[0],
num_face_nodes);
// Read face->node connectivity information.
int face_node_size = 0;
for (int i = 0; i < num_faces; ++i)
face_node_size += num_face_nodes[i];
int* face_nodes = polymec_malloc(sizeof(int) * face_node_size);
ex_get_conn(file->ex_id, EX_FACE_BLOCK, 1, face_nodes, NULL, NULL);
for (int i = 0; i < face_node_size; ++i)
face_nodes[i] -= 1;
fe_mesh_set_face_nodes(mesh, num_faces, num_face_nodes, face_nodes);
// Clean up.
polymec_free(num_face_nodes);
}
// Go over the element blocks and feel out the data.
for (int i = 0; i < file->num_elem_blocks; ++i)
{
int elem_block = file->elem_block_ids[i];
char elem_type_name[MAX_NAME_LENGTH+1];
int num_elem, num_nodes_per_elem, num_faces_per_elem;
ex_get_block(file->ex_id, EX_ELEM_BLOCK, elem_block,
elem_type_name, &num_elem,
&num_nodes_per_elem, NULL,
&num_faces_per_elem, NULL);
// Get the type of element for this block.
fe_mesh_element_t elem_type = get_element_type(elem_type_name);
fe_block_t* block = NULL;
char block_name[MAX_NAME_LENGTH+1];
if (elem_type == FE_POLYHEDRON)
{
// Find the number of faces for each element in the block.
int* num_elem_faces = polymec_malloc(sizeof(int) * num_elem);
ex_get_entity_count_per_polyhedra(file->ex_id, EX_ELEM_BLOCK, elem_block,
num_elem_faces);
// Get the element->face connectivity.
int elem_face_size = 0;
for (int j = 0; j < num_elem; ++j)
elem_face_size += num_elem_faces[j];
int* elem_faces = polymec_malloc(sizeof(int) * elem_face_size);
ex_get_conn(file->ex_id, EX_ELEM_BLOCK, elem_block, NULL, NULL, elem_faces);
// Subtract 1 from each element face.
for (int j = 0; j < elem_face_size; ++j)
elem_faces[j] -= 1;
// Create the element block.
block = polyhedral_fe_block_new(num_elem, num_elem_faces, elem_faces);
}
else if (elem_type != FE_INVALID)
{
// Get the element's nodal mapping.
int* node_conn = polymec_malloc(sizeof(int) * num_elem * num_nodes_per_elem);
ex_get_conn(file->ex_id, EX_ELEM_BLOCK, elem_block, node_conn, NULL, NULL);
// Subtract 1 from each element node.
for (int j = 0; j < num_elem * num_nodes_per_elem; ++j)
node_conn[j] -= 1;
// Build the element block.
block = fe_block_new(num_elem, elem_type, num_nodes_per_elem, node_conn);
}
else
{
fe_mesh_free(mesh);
ex_close(file->ex_id);
polymec_error("Block %d contains an invalid (3D) element type.", elem_block);
}
// Fish out the element block name if it has one, or make a default.
ex_get_name(file->ex_id, EX_ELEM_BLOCK, elem_block, block_name);
if (strlen(block_name) == 0)
sprintf(block_name, "block_%d", elem_block);
// Add the element block to the mesh.
fe_mesh_add_block(mesh, block_name, block);
}
// Fetch node positions and compute geometry.
real_t x[file->num_nodes], y[file->num_nodes], z[file->num_nodes];
ex_get_coord(file->ex_id, x, y, z);
point_t* X = fe_mesh_node_positions(mesh);
for (int n = 0; n < file->num_nodes; ++n)
{
X[n].x = x[n];
X[n].y = y[n];
X[n].z = z[n];
}
// Fetch sets of entities.
for (int i = 1; i <= file->num_elem_sets; ++i)
fetch_set(file, EX_ELEM_SET, i, mesh, fe_mesh_create_element_set);
for (int i = 1; i <= file->num_face_sets; ++i)
fetch_set(file, EX_FACE_SET, i, mesh, fe_mesh_create_face_set);
for (int i = 1; i <= file->num_edge_sets; ++i)
fetch_set(file, EX_EDGE_SET, i, mesh, fe_mesh_create_edge_set);
for (int i = 1; i <= file->num_node_sets; ++i)
fetch_set(file, EX_NODE_SET, i, mesh, fe_mesh_create_node_set);
for (int i = 1; i <= file->num_side_sets; ++i)
fetch_set(file, EX_SIDE_SET, i, mesh, fe_mesh_create_side_set);
return mesh;
}
static int exchanger_send_message(exchanger_t* ex, void* data, mpi_message_t* msg)
{
START_FUNCTION_TIMER();
// If we are expecting data, post asynchronous receives.
int j = 0;
for (int i = 0; i < msg->num_receives; ++i)
{
ASSERT(ex->rank != msg->source_procs[i]);
int err = MPI_Irecv(msg->receive_buffers[i],
msg->stride * msg->receive_buffer_sizes[i],
msg->type, msg->source_procs[i], msg->tag, ex->comm,
&(msg->requests[j++]));
if (err != MPI_SUCCESS)
{
int resultlen;
char str[MPI_MAX_ERROR_STRING];
MPI_Error_string(err, str, &resultlen);
char err_msg[1024];
snprintf(err_msg, 1024, "%d: MPI Error posting receive from %d: %d\n(%s)\n",
ex->rank, msg->source_procs[i], err, str);
polymec_error(err_msg);
}
}
// Send the data asynchronously.
for (int i = 0; i < msg->num_sends; ++i)
{
int err = MPI_Isend(msg->send_buffers[i],
msg->stride * msg->send_buffer_sizes[i],
msg->type, msg->dest_procs[i], msg->tag, ex->comm,
&(msg->requests[j++]));
if (err != MPI_SUCCESS)
{
int resultlen;
char str[MPI_MAX_ERROR_STRING];
MPI_Error_string(err, str, &resultlen);
char err_msg[1024];
snprintf(err_msg, 1024, "%d: MPI Error sending to %d: %d\n(%s)\n",
ex->rank, msg->dest_procs[i], err, str);
polymec_error(err_msg);
}
}
msg->num_requests = j;
// Allocate a token.
int token = 0;
while ((token < ex->num_pending_msgs) && (ex->pending_msgs[token] != NULL))
++token;
if (token == ex->num_pending_msgs) ++ex->num_pending_msgs;
if (ex->num_pending_msgs == ex->pending_msg_cap)
{
ex->pending_msg_cap *= 2;
ex->pending_msgs = polymec_realloc(ex->pending_msgs, ex->pending_msg_cap*sizeof(mpi_message_t));
ex->orig_buffers = polymec_realloc(ex->orig_buffers, ex->pending_msg_cap*sizeof(void*));
ex->transfer_counts = polymec_realloc(ex->transfer_counts, ex->pending_msg_cap*sizeof(int*));
}
ex->pending_msgs[token] = msg;
ex->orig_buffers[token] = data;
STOP_FUNCTION_TIMER();
return token;
}
sp_func_t* rect_prism_new(point_t* x0,
real_t L1, real_t L2, real_t L3,
real_t alpha, real_t beta, real_t gamma)
{
ASSERT(L1 > 0.0);
ASSERT(L2 > 0.0);
ASSERT(L3 > 0.0);
// FOR NOW, we disable Euler angles.
if ((alpha != 0.0) || (beta != 0.0) || (gamma != 0.0))
polymec_error("rect_prism_new: Euler angles not yet implemented!");
// Set the 6 bounding planes.
vector_t n;
point_t x;
sp_func_t* planes[6];
// "-x" direction
n.x = 1.0, n.y = 0.0, n.z = 0.0;
x.x = x0->x - 0.5*L1, x.y = x0->y, x.z = x0->z;
planes[0] = plane_new(&n, &x);
// "+x" direction
n.x = -1.0, n.y = 0.0, n.z = 0.0;
x.x = x0->x + 0.5*L1, x.y = x0->y, x.z = x0->z;
planes[1] = plane_new(&n, &x);
// "-y" direction
n.x = 0.0, n.y = 1.0, n.z = 0.0;
x.x = x0->x, x.y = x0->y - 0.5*L2, x.z = x0->z;
planes[2] = plane_new(&n, &x);
// "+y" direction
n.x = 0.0, n.y = -1.0, n.z = 0.0;
x.x = x0->x, x.y = x0->y + 0.5*L2, x.z = x0->z;
planes[3] = plane_new(&n, &x);
// "-z" direction
n.x = 0.0, n.y = 0.0, n.z = 1.0;
x.x = x0->x, x.y = x0->y, x.z = x0->z - 0.5*L3;
planes[4] = plane_new(&n, &x);
// "+y" direction
n.x = 0.0, n.y = 0.0, n.z = -1.0;
x.x = x0->x, x.y = x0->y, x.z = x0->z + 0.5*L3;
planes[5] = plane_new(&n, &x);
rect_prism_t* p = polymec_malloc(sizeof(rect_prism_t));
p->prism = intersection_new(planes, 6);
// Set up the spatial function.
sp_func_vtable vtable = {.eval = prism_eval,
.eval_deriv = prism_eval_deriv,
.has_deriv = prism_has_deriv,
.dtor = prism_free};
char str[1024];
snprintf(str, 1024, "Rectangular prism (x0 = (%g, %g, %g), L1 = %g, L2 = %g, L3 = %g,"
" alpha = %g, beta = %g, gamma = %g)",
x0->x, x0->y, x0->z, L1, L2, L3, alpha, beta, gamma);
return sp_func_new(str, p, vtable, SP_FUNC_INHOMOGENEOUS, 1);
}
