//-----------------------------------------------------------------------------
void BoundaryComputation::compute_boundary(const Mesh& mesh,
const std::string type,
BoundaryMesh& boundary)
{
// We iterate over all facets in the mesh and check if they are on
// the boundary. A facet is on the boundary if it is connected to
// exactly one cell.
log(TRACE, "Computing boundary mesh.");
bool exterior = true;
bool interior = true;
if (type == "exterior")
interior = false;
else if (type == "interior")
exterior = false;
else if (type != "local")
{
dolfin_error("BoundaryComputation.cpp",
"determine boundary mesh type",
"Unknown boundary type (%d)", type.c_str());
}
// Get my MPI process rank and number of MPI processes
const std::size_t my_rank = MPI::rank(mesh.mpi_comm());
const std::size_t num_processes = MPI::size(mesh.mpi_comm());
// Open boundary mesh for editing
const std::size_t D = mesh.topology().dim();
MeshEditor editor;
editor.open(boundary, mesh.type().facet_type(), D - 1, mesh.geometry().dim());
// Generate facet - cell connectivity if not generated
mesh.init(D - 1, D);
// Temporary arrays for assignment of indices to vertices on the boundary
std::map<std::size_t, std::size_t> boundary_vertices;
// Map of index "owners" (process responsible for assigning global index)
std::map< std::size_t, std::size_t > global_index_owner;
// Shared vertices for full mesh
// FIXME: const_cast
const std::map<unsigned int, std::set<unsigned int>> &
shared_vertices = const_cast<Mesh&>(mesh).topology().shared_entities(0);
// Shared vertices for boundary mesh
std::map<unsigned int, std::set<unsigned int>> shared_boundary_vertices;
if (exterior)
{
// Extract shared vertices if vertex is identified as part of globally
// exterior facet.
std::vector<std::size_t> boundary_global_indices;
for (std::map<unsigned int, std::set<unsigned int>>::const_iterator
sv_it=shared_vertices.begin(); sv_it != shared_vertices.end(); ++sv_it)
{
std::size_t local_mesh_index = sv_it->first;
Vertex v(mesh, local_mesh_index);
for (FacetIterator f(v); !f.end(); ++f)
{
if (f->num_global_entities(D) == 1)
{
const std::size_t global_mesh_index
= mesh.topology().global_indices(0)[local_mesh_index];
shared_boundary_vertices[local_mesh_index] = sv_it->second;
boundary_global_indices.push_back(global_mesh_index);
break;
}
}
}
// Distribute all shared boundary vertices
std::vector<std::vector<std::size_t>> boundary_global_indices_all;
MPI::all_gather(mesh.mpi_comm(), boundary_global_indices,
boundary_global_indices_all);
// Identify and clean up discrepancies between shared vertices of full mesh
// and shared vertices of boundary mesh
for (auto sbv_it = shared_boundary_vertices.begin();
sbv_it != shared_boundary_vertices.end(); )
{
std::size_t local_mesh_index = sbv_it->first;
const std::size_t global_mesh_index
= mesh.topology().global_indices(0)[local_mesh_index];
// Check if this vertex is identified as boundary vertex on
// other processes sharing this vertex
std::set<unsigned int> &other_processes = sbv_it->second;
for (auto op_it=other_processes.begin();
op_it != other_processes.end(); )
{
// Check if vertex is identified as boundary vertex on process *op_it
bool is_boundary_vertex
= (std::find(boundary_global_indices_all[*op_it].begin(),
boundary_global_indices_all[*op_it].end(),
global_mesh_index)
!= boundary_global_indices_all[*op_it].end());
// Erase item if this is not identified as a boundary vertex
// on process *op_it, and increment iterator
if (!is_boundary_vertex)
{
// Erase item while carefully avoiding invalidating the
// iterator: First increment it to get the next, valid
// iterator, and then erase what it pointed to from
// other_processes
other_processes.erase(op_it++);
}
else
++op_it;
}
// Erase item from map if no other processes identify this
// vertex as a boundary vertex, and increment iterator
if (other_processes.size() == 0)
{
// Erase carefully as above
shared_boundary_vertices.erase(sbv_it++);
}
else
++sbv_it;
}
}
else
{
// If interior boundary, shared vertices are the same
shared_boundary_vertices = shared_vertices;
}
// Determine boundary facet, count boundary vertices and facets, and
// assign vertex indices
std::size_t num_boundary_vertices = 0;
std::size_t num_owned_vertices = 0;
std::size_t num_boundary_cells = 0;
MeshFunction<bool> boundary_facet(reference_to_no_delete_pointer(mesh),
D - 1, false);
for (FacetIterator f(mesh); !f.end(); ++f)
{
// Boundary facets are connected to exactly one cell
if (f->num_entities(D) == 1)
{
const bool global_exterior_facet = (f->num_global_entities(D) == 1);
if (global_exterior_facet && exterior)
boundary_facet[*f] = true;
else if (!global_exterior_facet && interior)
boundary_facet[*f] = true;
if (boundary_facet[*f])
{
// Count boundary vertices and assign indices
for (VertexIterator v(*f); !v.end(); ++v)
{
const std::size_t local_mesh_index = v->index();
if (boundary_vertices.find(local_mesh_index)
== boundary_vertices.end())
{
const std::size_t local_boundary_index = num_boundary_vertices;
boundary_vertices[local_mesh_index] = local_boundary_index;
// Determine "owner" of global_mesh_index
std::size_t owner = my_rank;
std::map<unsigned int, std::set<unsigned int>>::const_iterator
other_processes_it
= shared_boundary_vertices.find(local_mesh_index);
if (other_processes_it != shared_boundary_vertices.end() && D > 1)
{
const std::set<unsigned int>& other_processes
= other_processes_it->second;
const std::size_t min_process
= *std::min_element(other_processes.begin(),
other_processes.end());
boundary.topology().shared_entities(0)[local_boundary_index]
= other_processes;
// FIXME: More sophisticated ownership determination
if (min_process < owner)
owner = min_process;
}
const std::size_t global_mesh_index
= mesh.topology().global_indices(0)[local_mesh_index];
global_index_owner[global_mesh_index] = owner;
// Update counts
if (owner == my_rank)
num_owned_vertices++;
num_boundary_vertices++;
}
}
// Count boundary cells (facets of the mesh)
num_boundary_cells++;
}
}
}
// Initiate boundary topology
/*
boundary.topology().init(0, num_boundary_vertices,
MPI::sum(mesh.mpi_comm(), num_owned_vertices));
boundary.topology().init(D - 1, num_boundary_cells,
MPI::sum(mesh.mpi_comm(), num_boundary_cells));
*/
// Specify number of vertices and cells
editor.init_vertices_global(num_boundary_vertices,
MPI::sum(mesh.mpi_comm(), num_owned_vertices));
editor.init_cells_global(num_boundary_cells, MPI::sum(mesh.mpi_comm(),
num_boundary_cells));
// Write vertex map
MeshFunction<std::size_t>& vertex_map = boundary.entity_map(0);
if (num_boundary_vertices > 0)
{
vertex_map.init(reference_to_no_delete_pointer(boundary), 0,
num_boundary_vertices);
}
std::map<std::size_t, std::size_t>::const_iterator it;
for (it = boundary_vertices.begin(); it != boundary_vertices.end(); ++it)
vertex_map[it->second] = it->first;
// Get vertex ownership distribution, and find index to start global
// numbering from
std::vector<std::size_t> ownership_distribution(num_processes);
MPI::all_gather(mesh.mpi_comm(), num_owned_vertices, ownership_distribution);
std::size_t start_index = 0;
for (std::size_t j = 0; j < my_rank; j++)
start_index += ownership_distribution[j];
// Set global indices of owned vertices, request global indices for
// vertices owned elsewhere
std::map<std::size_t, std::size_t> global_indices;
std::vector<std::vector<std::size_t>> request_global_indices(num_processes);
std::size_t current_index = start_index;
for (std::size_t local_boundary_index = 0;
local_boundary_index<num_boundary_vertices; local_boundary_index++)
{
const std::size_t local_mesh_index = vertex_map[local_boundary_index];
const std::size_t global_mesh_index
= mesh.topology().global_indices(0)[local_mesh_index];
const std::size_t owner = global_index_owner[global_mesh_index];
if (owner != my_rank)
request_global_indices[owner].push_back(global_mesh_index);
else
global_indices[global_mesh_index] = current_index++;
}
// Send and receive requests from other processes
std::vector<std::vector<std::size_t>> global_index_requests(num_processes);
MPI::all_to_all(mesh.mpi_comm(), request_global_indices,
global_index_requests);
// Find response to requests of global indices
std::vector<std::vector<std::size_t>> respond_global_indices(num_processes);
for (std::size_t i = 0; i < num_processes; i++)
{
const std::size_t N = global_index_requests[i].size();
respond_global_indices[i].resize(N);
for (std::size_t j = 0; j < N; j++)
respond_global_indices[i][j]
= global_indices[global_index_requests[i][j]];
}
// Scatter responses back to requesting processes
std::vector<std::vector<std::size_t>> global_index_responses(num_processes);
MPI::all_to_all(mesh.mpi_comm(), respond_global_indices,
global_index_responses);
// Update global_indices
for (std::size_t i = 0; i < num_processes; i++)
{
const std::size_t N = global_index_responses[i].size();
// Check that responses are the same size as the requests made
dolfin_assert(global_index_responses[i].size()
== request_global_indices[i].size());
for (std::size_t j = 0; j < N; j++)
{
const std::size_t global_mesh_index = request_global_indices[i][j];
const std::size_t global_boundary_index = global_index_responses[i][j];
global_indices[global_mesh_index] = global_boundary_index;
}
}
// Create vertices
for (std::size_t local_boundary_index = 0;
local_boundary_index < num_boundary_vertices; local_boundary_index++)
{
const std::size_t local_mesh_index = vertex_map[local_boundary_index];
const std::size_t global_mesh_index
= mesh.topology().global_indices(0)[local_mesh_index];
const std::size_t global_boundary_index = global_indices[global_mesh_index];
Vertex v(mesh, local_mesh_index);
editor.add_vertex_global(local_boundary_index, global_boundary_index,
v.point());
}
// Find global index to start cell numbering from for current process
std::vector<std::size_t> cell_distribution(num_processes);
MPI::all_gather(mesh.mpi_comm(), num_boundary_cells, cell_distribution);
std::size_t start_cell_index = 0;
for (std::size_t i = 0; i < my_rank; i++)
start_cell_index += cell_distribution[i];
// Create cells (facets) and map between boundary mesh cells and facets parent
MeshFunction<std::size_t>& cell_map = boundary.entity_map(D - 1);
if (num_boundary_cells > 0)
{
cell_map.init(reference_to_no_delete_pointer(boundary), D - 1,
num_boundary_cells);
}
std::vector<std::size_t>
cell(boundary.type().num_vertices(boundary.topology().dim()));
std::size_t current_cell = 0;
for (FacetIterator f(mesh); !f.end(); ++f)
{
if (boundary_facet[*f])
{
// Compute new vertex numbers for cell
const unsigned int* vertices = f->entities(0);
for (std::size_t i = 0; i < cell.size(); i++)
cell[i] = boundary_vertices[vertices[i]];
// Reorder vertices so facet is right-oriented w.r.t. facet
// normal
reorder(cell, *f);
// Create mapping from boundary cell to mesh facet if requested
if (!cell_map.empty())
cell_map[current_cell] = f->index();
// Add cell
editor.add_cell(current_cell, start_cell_index+current_cell, cell);
current_cell++;
}
}
// Close mesh editor. Note the argument order=false to prevent
// ordering from destroying the orientation of facets accomplished
// by calling reorder() below.
editor.close(false);
}
//-----------------------------------------------------------------------------
void MeshPartitioning::build_mesh(Mesh& mesh,
const std::vector<std::size_t>& global_cell_indices,
const boost::multi_array<std::size_t, 2>& cell_global_vertices,
const std::vector<std::size_t>& vertex_indices,
const boost::multi_array<double, 2>& vertex_coordinates,
const std::map<std::size_t, std::size_t>& vertex_global_to_local,
std::size_t tdim, std::size_t gdim, std::size_t num_global_cells,
std::size_t num_global_vertices)
{
Timer timer("PARALLEL 3: Build mesh (from local mesh data)");
// Get number of processes and process number
const std::size_t num_processes = MPI::num_processes();
const std::size_t process_number = MPI::process_number();
// Open mesh for editing
mesh.clear();
MeshEditor editor;
editor.open(mesh, tdim, gdim);
// Add vertices
editor.init_vertices(vertex_coordinates.size());
Point point(gdim);
dolfin_assert(vertex_indices.size() == vertex_coordinates.size());
for (std::size_t i = 0; i < vertex_coordinates.size(); ++i)
{
for (std::size_t j = 0; j < gdim; ++j)
point[j] = vertex_coordinates[i][j];
editor.add_vertex_global(i, vertex_indices[i], point);
}
// Add cells
editor.init_cells(cell_global_vertices.size());
const std::size_t num_cell_vertices = tdim + 1;
std::vector<std::size_t> cell(num_cell_vertices);
for (std::size_t i = 0; i < cell_global_vertices.size(); ++i)
{
for (std::size_t j = 0; j < num_cell_vertices; ++j)
{
// Get local cell vertex
std::map<std::size_t, std::size_t>::const_iterator iter
= vertex_global_to_local.find(cell_global_vertices[i][j]);
dolfin_assert(iter != vertex_global_to_local.end());
cell[j] = iter->second;
}
editor.add_cell(i, global_cell_indices[i], cell);
}
// Close mesh: Note that this must be done after creating the global
// vertex map or otherwise the ordering in mesh.close() will be wrong
// (based on local numbers).
editor.close();
// Set global number of cells and vertices
mesh.topology().init_global(0, num_global_vertices);
mesh.topology().init_global(tdim, num_global_cells);
// Construct boundary mesh
BoundaryMesh bmesh(mesh, "exterior");
const MeshFunction<std::size_t>& boundary_vertex_map = bmesh.entity_map(0);
const std::size_t boundary_size = boundary_vertex_map.size();
// Build sorted array of global boundary vertex indices (global
// numbering)
std::vector<std::size_t> global_vertex_send(boundary_size);
for (std::size_t i = 0; i < boundary_size; ++i)
global_vertex_send[i] = vertex_indices[boundary_vertex_map[i]];
std::sort(global_vertex_send.begin(), global_vertex_send.end());
// Receive buffer
std::vector<std::size_t> global_vertex_recv;
// Create shared_vertices data structure: mapping from shared vertices
// to list of neighboring processes
std::map<unsigned int, std::set<unsigned int> >& shared_vertices
= mesh.topology().shared_entities(0);
shared_vertices.clear();
// FIXME: Remove computation from inside communication loop
// Build shared vertex to sharing processes map
for (std::size_t i = 1; i < num_processes; ++i)
{
// We send data to process p - i (i steps to the left)
const int p = (process_number - i + num_processes) % num_processes;
// We receive data from process p + i (i steps to the right)
const int q = (process_number + i) % num_processes;
// Send and receive
MPI::send_recv(global_vertex_send, p, global_vertex_recv, q);
// Compute intersection of global indices
std::vector<std::size_t> intersection(std::min(global_vertex_send.size(),
global_vertex_recv.size()));
std::vector<std::size_t>::iterator intersection_end
= std::set_intersection(global_vertex_send.begin(),
global_vertex_send.end(),
global_vertex_recv.begin(),
global_vertex_recv.end(),
intersection.begin());
// Fill shared vertices information
std::vector<std::size_t>::const_iterator global_index;
for (global_index = intersection.begin(); global_index != intersection_end;
++global_index)
{
// Get local index
std::map<std::size_t, std::size_t>::const_iterator local_index;
local_index = vertex_global_to_local.find(*global_index);
dolfin_assert(local_index != vertex_global_to_local.end());
// Insert (local index, [proc])
shared_vertices[local_index->second].insert(q);
}
}
}