void
MaxQpsThread::operator()(const ConstElemRange & range)
{
ParallelUniqueId puid;
_tid = puid.id;
// For short circuiting reinit
std::set<ElemType> seen_it;
for (const auto & elem : range)
{
// Only reinit if the element type has not previously been seen
if (seen_it.insert(elem->type()).second)
{
FEType fe_type(FIRST, LAGRANGE);
unsigned int dim = elem->dim();
unsigned int side = 0; // we assume that any element will have at least one side ;)
// We cannot mess with the FE objects in Assembly, because we might need to request second
// derivatives
// later on. If we used them, we'd call reinit on them, thus making the call to request second
// derivatives harmful (i.e. leading to segfaults/asserts). Thus, we have to use a locally
// allocated object here.
std::unique_ptr<FEBase> fe(FEBase::build(dim, fe_type));
// figure out the number of qps for volume
std::unique_ptr<QBase> qrule(QBase::build(_qtype, dim, _order));
fe->attach_quadrature_rule(qrule.get());
fe->reinit(elem);
if (qrule->n_points() > _max)
_max = qrule->n_points();
unsigned int n_shape_funcs = fe->n_shape_functions();
if (n_shape_funcs > _max_shape_funcs)
_max_shape_funcs = n_shape_funcs;
// figure out the number of qps for the face
// NOTE: user might specify higher order rule for faces, thus possibly ending up with more qps
// than in the volume
std::unique_ptr<QBase> qrule_face(QBase::build(_qtype, dim - 1, _face_order));
fe->attach_quadrature_rule(qrule_face.get());
fe->reinit(elem, side);
if (qrule_face->n_points() > _max)
_max = qrule_face->n_points();
}
}
}
void
MultiAppProjectionTransfer::assembleL2From(EquationSystems & es, const std::string & system_name)
{
unsigned int n_apps = _multi_app->numGlobalApps();
std::vector<NumericVector<Number> *> from_slns(n_apps, NULL);
std::vector<MeshFunction *> from_fns(n_apps, NULL);
std::vector<MeshTools::BoundingBox *> from_bbs(n_apps, NULL);
// get bounding box, mesh function and solution for each subapp
for (unsigned int i = 0; i < n_apps; i++)
{
if (!_multi_app->hasLocalApp(i))
continue;
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
FEProblem & from_problem = *_multi_app->appProblem(i);
EquationSystems & from_es = from_problem.es();
MeshBase & from_mesh = from_es.get_mesh();
MeshTools::BoundingBox * app_box = new MeshTools::BoundingBox(MeshTools::processor_bounding_box(from_mesh, from_mesh.processor_id()));
from_bbs[i] = app_box;
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
System & from_sys = from_var.sys().system();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
NumericVector<Number> * serialized_from_solution = NumericVector<Number>::build(from_sys.comm()).release();
serialized_from_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_from_solution);
from_slns[i] = serialized_from_solution;
MeshFunction * from_func = new MeshFunction(from_es, *serialized_from_solution, from_sys.get_dof_map(), from_var_num);
from_func->init(Trees::ELEMENTS);
from_func->enable_out_of_mesh_mode(NOTFOUND);
from_fns[i] = from_func;
Moose::swapLibMeshComm(swapped);
}
const MeshBase& mesh = es.get_mesh();
const unsigned int dim = mesh.mesh_dimension();
LinearImplicitSystem & system = es.get_system<LinearImplicitSystem>(system_name);
FEType fe_type = system.variable_type(0);
AutoPtr<FEBase> fe(FEBase::build(dim, fe_type));
QGauss qrule(dim, fe_type.default_quadrature_order());
fe->attach_quadrature_rule(&qrule);
const std::vector<Real> & JxW = fe->get_JxW();
const std::vector<std::vector<Real> > & phi = fe->get_phi();
const std::vector<Point> & xyz = fe->get_xyz();
const DofMap& dof_map = system.get_dof_map();
DenseMatrix<Number> Ke;
DenseVector<Number> Fe;
std::vector<dof_id_type> dof_indices;
MeshBase::const_element_iterator el = mesh.active_local_elements_begin();
const MeshBase::const_element_iterator end_el = mesh.active_local_elements_end();
for ( ; el != end_el; ++el)
{
const Elem* elem = *el;
fe->reinit (elem);
dof_map.dof_indices (elem, dof_indices);
Ke.resize (dof_indices.size(), dof_indices.size());
Fe.resize (dof_indices.size());
for (unsigned int qp = 0; qp < qrule.n_points(); qp++)
{
Point qpt = xyz[qp];
Real f = 0.;
for (unsigned int app = 0; app < n_apps; app++)
{
Point pt = qpt - _multi_app->position(app);
if (from_bbs[app] != NULL && from_bbs[app]->contains_point(pt))
{
MPI_Comm swapped = Moose::swapLibMeshComm(_multi_app->comm());
f = (*from_fns[app])(pt);
Moose::swapLibMeshComm(swapped);
break;
}
}
// Now compute the element matrix and RHS contributions.
for (unsigned int i=0; i<phi.size(); i++)
{
// RHS
Fe(i) += JxW[qp] * (f * phi[i][qp]);
if (_compute_matrix)
for (unsigned int j = 0; j < phi.size(); j++)
{
// The matrix contribution
Ke(i,j) += JxW[qp] * (phi[i][qp] * phi[j][qp]);
}
}
dof_map.constrain_element_matrix_and_vector(Ke, Fe, dof_indices);
if (_compute_matrix)
system.matrix->add_matrix(Ke, dof_indices);
system.rhs->add_vector(Fe, dof_indices);
}
}
for (unsigned int i = 0; i < n_apps; i++)
{
delete from_fns[i];
delete from_bbs[i];
delete from_slns[i];
}
}
void
MultiAppProjectionTransfer::assembleL2To(EquationSystems & es, const std::string & system_name)
{
unsigned int app = es.parameters.get<unsigned int>("app");
FEProblem & from_problem = *_multi_app->problem();
EquationSystems & from_es = from_problem.es();
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
System & from_sys = from_var.sys().system();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
NumericVector<Number> * serialized_from_solution = NumericVector<Number>::build(from_sys.comm()).release();
serialized_from_solution->init(from_sys.n_dofs(), false, SERIAL);
// Need to pull down a full copy of this vector on every processor so we can get values in parallel
from_sys.solution->localize(*serialized_from_solution);
MeshFunction from_func(from_es, *serialized_from_solution, from_sys.get_dof_map(), from_var_num);
from_func.init(Trees::ELEMENTS);
from_func.enable_out_of_mesh_mode(0.);
const MeshBase& mesh = es.get_mesh();
const unsigned int dim = mesh.mesh_dimension();
LinearImplicitSystem & system = es.get_system<LinearImplicitSystem>(system_name);
FEType fe_type = system.variable_type(0);
AutoPtr<FEBase> fe(FEBase::build(dim, fe_type));
QGauss qrule(dim, fe_type.default_quadrature_order());
fe->attach_quadrature_rule(&qrule);
const std::vector<Real> & JxW = fe->get_JxW();
const std::vector<std::vector<Real> > & phi = fe->get_phi();
const std::vector<Point> & xyz = fe->get_xyz();
const DofMap& dof_map = system.get_dof_map();
DenseMatrix<Number> Ke;
DenseVector<Number> Fe;
std::vector<dof_id_type> dof_indices;
MeshBase::const_element_iterator el = mesh.active_local_elements_begin();
const MeshBase::const_element_iterator end_el = mesh.active_local_elements_end();
for ( ; el != end_el; ++el)
{
const Elem* elem = *el;
fe->reinit (elem);
dof_map.dof_indices (elem, dof_indices);
Ke.resize (dof_indices.size(), dof_indices.size());
Fe.resize (dof_indices.size());
for (unsigned int qp = 0; qp < qrule.n_points(); qp++)
{
Point qpt = xyz[qp];
Point pt = qpt + _multi_app->position(app);
Real f = from_func(pt);
// Now compute the element matrix and RHS contributions.
for (unsigned int i=0; i<phi.size(); i++)
{
// RHS
Fe(i) += JxW[qp] * (f * phi[i][qp]);
if (_compute_matrix)
for (unsigned int j = 0; j < phi.size(); j++)
{
// The matrix contribution
Ke(i,j) += JxW[qp] * (phi[i][qp] * phi[j][qp]);
}
}
dof_map.constrain_element_matrix_and_vector(Ke, Fe, dof_indices);
if (_compute_matrix)
system.matrix->add_matrix(Ke, dof_indices);
system.rhs->add_vector(Fe, dof_indices);
}
}
}
void
MultiAppProjectionTransfer::execute()
{
_console << "Beginning projection transfer " << name() << std::endl;
getAppInfo();
////////////////////
// We are going to project the solutions by solving some linear systems. In
// order to assemble the systems, we need to evaluate the "from" domain
// solutions at quadrature points in the "to" domain. Some parallel
// communication is necessary because each processor doesn't necessarily have
// all the "from" information it needs to set its "to" values. We don't want
// to use a bunch of big all-to-all broadcasts, so we'll use bounding boxes to
// figure out which processors have the information we need and only
// communicate with those processors.
//
// Each processor will
// 1. Check its local quadrature points in the "to" domains to see which
// "from" domains they might be in.
// 2. Send quadrature points to the processors with "from" domains that might
// contain those points.
// 3. Recieve quadrature points from other processors, evaluate its mesh
// functions at those points, and send the values back to the proper
// processor
// 4. Recieve mesh function evaluations from all relevant processors and
// decide which one to use at every quadrature point (the lowest global app
// index always wins)
// 5. And use the mesh function evaluations to assemble and solve an L2
// projection system on its local elements.
////////////////////
////////////////////
// For every combination of global "from" problem and local "to" problem, find
// which "from" bounding boxes overlap with which "to" elements. Keep track
// of which processors own bounding boxes that overlap with which elements.
// Build vectors of quadrature points to send to other processors for mesh
// function evaluations.
////////////////////
// Get the bounding boxes for the "from" domains.
std::vector<MeshTools::BoundingBox> bboxes = getFromBoundingBoxes();
// Figure out how many "from" domains each processor owns.
std::vector<unsigned int> froms_per_proc = getFromsPerProc();
std::vector<std::vector<Point> > outgoing_qps(n_processors());
std::vector<std::map<std::pair<unsigned int, unsigned int>, unsigned int> > element_index_map(n_processors());
// element_index_map[i_to, element_id] = index
// outgoing_qps[index] is the first quadrature point in element
if (! _qps_cached)
{
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
MeshBase & to_mesh = _to_meshes[i_to]->getMesh();
LinearImplicitSystem & system = * _proj_sys[i_to];
FEType fe_type = system.variable_type(0);
std::unique_ptr<FEBase> fe(FEBase::build(to_mesh.mesh_dimension(), fe_type));
QGauss qrule(to_mesh.mesh_dimension(), fe_type.default_quadrature_order());
fe->attach_quadrature_rule(&qrule);
const std::vector<Point> & xyz = fe->get_xyz();
MeshBase::const_element_iterator el = to_mesh.local_elements_begin();
const MeshBase::const_element_iterator end_el = to_mesh.local_elements_end();
unsigned int from0 = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors();
from0 += froms_per_proc[i_proc], i_proc++)
{
for (el = to_mesh.local_elements_begin(); el != end_el; el++)
{
const Elem* elem = *el;
fe->reinit (elem);
bool qp_hit = false;
for (unsigned int i_from = 0;
i_from < froms_per_proc[i_proc] && ! qp_hit; i_from++)
{
for (unsigned int qp = 0;
qp < qrule.n_points() && ! qp_hit; qp ++)
{
Point qpt = xyz[qp];
if (bboxes[from0 + i_from].contains_point(qpt + _to_positions[i_to]))
qp_hit = true;
}
}
if (qp_hit)
{
// The selected processor's bounding box contains at least one
// quadrature point from this element. Send all qps from this element
// and remember where they are in the array using the map.
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
element_index_map[i_proc][key] = outgoing_qps[i_proc].size();
for (unsigned int qp = 0; qp < qrule.n_points(); qp ++)
{
Point qpt = xyz[qp];
outgoing_qps[i_proc].push_back(qpt + _to_positions[i_to]);
}
}
}
}
}
if (_fixed_meshes)
_cached_index_map = element_index_map;
}
else
{
element_index_map = _cached_index_map;
}
////////////////////
// Request quadrature point evaluations from other processors and handle
// requests sent to this processor.
////////////////////
// Non-blocking send quadrature points to other processors.
std::vector<Parallel::Request> send_qps(n_processors());
if (! _qps_cached)
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
if (i_proc != processor_id())
_communicator.send(i_proc, outgoing_qps[i_proc], send_qps[i_proc]);
// Get the local bounding boxes.
std::vector<MeshTools::BoundingBox> local_bboxes(froms_per_proc[processor_id()]);
{
// Find the index to the first of this processor's local bounding boxes.
unsigned int local_start = 0;
for (processor_id_type i_proc = 0;
i_proc < n_processors() && i_proc != processor_id();
i_proc++)
local_start += froms_per_proc[i_proc];
// Extract the local bounding boxes.
for (unsigned int i_from = 0; i_from < froms_per_proc[processor_id()]; i_from++)
local_bboxes[i_from] = bboxes[local_start + i_from];
}
// Setup the local mesh functions.
std::vector<MeshFunction *> local_meshfuns(froms_per_proc[processor_id()], NULL);
for (unsigned int i_from = 0; i_from < _from_problems.size(); i_from++)
{
FEProblemBase & from_problem = *_from_problems[i_from];
MooseVariable & from_var = from_problem.getVariable(0, _from_var_name);
System & from_sys = from_var.sys().system();
unsigned int from_var_num = from_sys.variable_number(from_var.name());
MeshFunction * from_func = new MeshFunction(from_problem.es(),
*from_sys.current_local_solution, from_sys.get_dof_map(), from_var_num);
from_func->init(Trees::ELEMENTS);
from_func->enable_out_of_mesh_mode(OutOfMeshValue);
local_meshfuns[i_from] = from_func;
}
// Recieve quadrature points from other processors, evaluate mesh frunctions
// at those points, and send the values back.
std::vector<Parallel::Request> send_evals(n_processors());
std::vector<Parallel::Request> send_ids(n_processors());
std::vector<std::vector<Real> > outgoing_evals(n_processors());
std::vector<std::vector<unsigned int> > outgoing_ids(n_processors());
std::vector<std::vector<Real> > incoming_evals(n_processors());
std::vector<std::vector<unsigned int> > incoming_app_ids(n_processors());
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
// Use the cached qps if they're available.
std::vector<Point> incoming_qps;
if (! _qps_cached)
{
if (i_proc == processor_id())
incoming_qps = outgoing_qps[i_proc];
else
_communicator.receive(i_proc, incoming_qps);
// Cache these qps for later if _fixed_meshes
if (_fixed_meshes)
_cached_qps[i_proc] = incoming_qps;
}
else
{
incoming_qps = _cached_qps[i_proc];
}
outgoing_evals[i_proc].resize(incoming_qps.size(), OutOfMeshValue);
if (_direction == FROM_MULTIAPP)
outgoing_ids[i_proc].resize(incoming_qps.size(), libMesh::invalid_uint);
for (unsigned int qp = 0; qp < incoming_qps.size(); qp++)
{
Point qpt = incoming_qps[qp];
// Loop until we've found the lowest-ranked app that actually contains
// the quadrature point.
for (unsigned int i_from = 0; i_from < _from_problems.size(); i_from++)
{
if (local_bboxes[i_from].contains_point(qpt))
{
outgoing_evals[i_proc][qp] = (* local_meshfuns[i_from])(qpt - _from_positions[i_from]);
if (_direction == FROM_MULTIAPP)
outgoing_ids[i_proc][qp] = _local2global_map[i_from];
}
}
}
if (i_proc == processor_id())
{
incoming_evals[i_proc] = outgoing_evals[i_proc];
if (_direction == FROM_MULTIAPP)
incoming_app_ids[i_proc] = outgoing_ids[i_proc];
}
else
{
_communicator.send(i_proc, outgoing_evals[i_proc], send_evals[i_proc]);
if (_direction == FROM_MULTIAPP)
_communicator.send(i_proc, outgoing_ids[i_proc], send_ids[i_proc]);
}
}
////////////////////
// Gather all of the qp evaluations and pick out the best ones for each qp.
////////////////////
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
_communicator.receive(i_proc, incoming_evals[i_proc]);
if (_direction == FROM_MULTIAPP)
_communicator.receive(i_proc, incoming_app_ids[i_proc]);
}
std::vector<std::vector<Real> > final_evals(_to_problems.size());
std::vector<std::map<unsigned int, unsigned int> > trimmed_element_maps(_to_problems.size());
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
MeshBase & to_mesh = _to_meshes[i_to]->getMesh();
LinearImplicitSystem & system = * _proj_sys[i_to];
FEType fe_type = system.variable_type(0);
std::unique_ptr<FEBase> fe(FEBase::build(to_mesh.mesh_dimension(), fe_type));
QGauss qrule(to_mesh.mesh_dimension(), fe_type.default_quadrature_order());
fe->attach_quadrature_rule(&qrule);
const std::vector<Point> & xyz = fe->get_xyz();
MeshBase::const_element_iterator el = to_mesh.local_elements_begin();
const MeshBase::const_element_iterator end_el = to_mesh.local_elements_end();
for (el = to_mesh.active_local_elements_begin(); el != end_el; el++)
{
const Elem* elem = *el;
fe->reinit (elem);
bool element_is_evaled = false;
std::vector<Real> evals(qrule.n_points(), 0.);
for (unsigned int qp = 0; qp < qrule.n_points(); qp++)
{
Point qpt = xyz[qp];
unsigned int lowest_app_rank = libMesh::invalid_uint;
for (unsigned int i_proc = 0; i_proc < n_processors(); i_proc++)
{
// Ignore the selected processor if the element wasn't found in it's
// bounding box.
std::map<std::pair<unsigned int, unsigned int>, unsigned int> & map = element_index_map[i_proc];
std::pair<unsigned int, unsigned int> key(i_to, elem->id());
if (map.find(key) == map.end())
continue;
unsigned int qp0 = map[key];
// Ignore the selected processor if it's app has a higher rank than the
// previously found lowest app rank.
if (_direction == FROM_MULTIAPP)
if (incoming_app_ids[i_proc][qp0 + qp] >= lowest_app_rank)
continue;
// Ignore the selected processor if the qp was actually outside the
// processor's subapp's mesh.
if (incoming_evals[i_proc][qp0 + qp] == OutOfMeshValue)
continue;
// This is the best meshfunction evaluation so far, save it.
element_is_evaled = true;
evals[qp] = incoming_evals[i_proc][qp0 + qp];
}
}
// If we found good evaluations for any of the qps in this element, save
// those evaluations for later.
if (element_is_evaled)
{
trimmed_element_maps[i_to][elem->id()] = final_evals[i_to].size();
for (unsigned int qp = 0; qp < qrule.n_points(); qp++)
final_evals[i_to].push_back(evals[qp]);
}
}
}
////////////////////
// We now have just one or zero mesh function values at all of our local
// quadrature points. Stash those values (and a map linking them to element
// ids) in the equation systems parameters and project the solution.
////////////////////
for (unsigned int i_to = 0; i_to < _to_problems.size(); i_to++)
{
_to_es[i_to]->parameters.set<std::vector<Real>*>("final_evals") = & final_evals[i_to];
_to_es[i_to]->parameters.set<std::map<unsigned int, unsigned int>*>("element_map") = & trimmed_element_maps[i_to];
projectSolution(i_to);
_to_es[i_to]->parameters.set<std::vector<Real>*>("final_evals") = NULL;
_to_es[i_to]->parameters.set<std::map<unsigned int, unsigned int>*>("element_map") = NULL;
}
for (unsigned int i = 0; i < _from_problems.size(); i++)
delete local_meshfuns[i];
// Make sure all our sends succeeded.
for (processor_id_type i_proc = 0; i_proc < n_processors(); i_proc++)
{
if (i_proc == processor_id())
continue;
if (! _qps_cached)
send_qps[i_proc].wait();
send_evals[i_proc].wait();
if (_direction == FROM_MULTIAPP)
send_ids[i_proc].wait();
}
if (_fixed_meshes)
_qps_cached = true;
_console << "Finished projection transfer " << name() << std::endl;
}
void
MultiAppProjectionTransfer::assembleL2(EquationSystems & es, const std::string & system_name)
{
// Get the system and mesh from the input arguments.
LinearImplicitSystem & system = es.get_system<LinearImplicitSystem>(system_name);
MeshBase & to_mesh = es.get_mesh();
// Get the meshfunction evaluations and the map that was stashed in the es.
std::vector<Real> & final_evals = * es.parameters.get<std::vector<Real>*>("final_evals");
std::map<unsigned int, unsigned int> & element_map = * es.parameters.get<std::map<unsigned int, unsigned int>*>("element_map");
// Setup system vectors and matrices.
FEType fe_type = system.variable_type(0);
std::unique_ptr<FEBase> fe(FEBase::build(to_mesh.mesh_dimension(), fe_type));
QGauss qrule(to_mesh.mesh_dimension(), fe_type.default_quadrature_order());
fe->attach_quadrature_rule(&qrule);
const DofMap& dof_map = system.get_dof_map();
DenseMatrix<Number> Ke;
DenseVector<Number> Fe;
std::vector<dof_id_type> dof_indices;
const std::vector<Real> & JxW = fe->get_JxW();
const std::vector<std::vector<Real> > & phi = fe->get_phi();
const MeshBase::const_element_iterator end_el = to_mesh.active_local_elements_end();
for (MeshBase::const_element_iterator el = to_mesh.active_local_elements_begin(); el != end_el; ++el)
{
const Elem* elem = *el;
fe->reinit (elem);
dof_map.dof_indices (elem, dof_indices);
Ke.resize (dof_indices.size(), dof_indices.size());
Fe.resize (dof_indices.size());
for (unsigned int qp = 0; qp < qrule.n_points(); qp++)
{
Real meshfun_eval = 0.;
if (element_map.find(elem->id()) != element_map.end())
{
// We have evaluations for this element.
meshfun_eval = final_evals[element_map[elem->id()] + qp];
}
// Now compute the element matrix and RHS contributions.
for (unsigned int i=0; i<phi.size(); i++)
{
// RHS
Fe(i) += JxW[qp] * (meshfun_eval * phi[i][qp]);
if (_compute_matrix)
for (unsigned int j = 0; j < phi.size(); j++)
{
// The matrix contribution
Ke(i,j) += JxW[qp] * (phi[i][qp] * phi[j][qp]);
}
}
dof_map.constrain_element_matrix_and_vector(Ke, Fe, dof_indices);
if (_compute_matrix)
system.matrix->add_matrix(Ke, dof_indices);
system.rhs->add_vector(Fe, dof_indices);
}
}
}
int main(int argc, char **argv)
{
#ifdef QUESO_HAVE_LIBMESH
unsigned int i, j;
QUESO::EnvOptionsValues opts;
opts.m_seed = -1;
#ifdef QUESO_HAS_MPI
MPI_Init(&argc, &argv);
QUESO::FullEnvironment env(MPI_COMM_WORLD, "", "", &opts);
#else
QUESO::FullEnvironment env("", "", &opts);
#endif
#ifdef LIBMESH_DEFAULT_SINGLE_PRECISION
// SLEPc currently gives us a nasty crash with Real==float
libmesh_example_assert(false, "--disable-singleprecision");
#endif
// Need an artificial block here because libmesh needs to
// call PetscFinalize before we call MPI_Finalize
#ifdef LIBMESH_HAVE_SLEPC
{
libMesh::LibMeshInit init(argc, argv);
libMesh::Mesh mesh(init.comm());
libMesh::MeshTools::Generation::build_square(mesh,
20, 20, 0.0, 1.0, 0.0, 1.0, libMeshEnums::QUAD4);
QUESO::FunctionOperatorBuilder builder;
builder.order = "FIRST";
builder.family = "LAGRANGE";
builder.num_req_eigenpairs = 10;
QUESO::LibMeshNegativeLaplacianOperator precision(builder, mesh);
libMesh::EquationSystems & es = precision.get_equation_systems();
libMesh::CondensedEigenSystem & eig_sys = es.get_system<libMesh::CondensedEigenSystem>(
"Eigensystem");
// Check all eigenfunctions have unit L2 norm
std::vector<double> norms(builder.num_req_eigenpairs, 0);
for (i = 0; i < builder.num_req_eigenpairs; i++) {
eig_sys.get_eigenpair(i);
norms[i] = eig_sys.calculate_norm(*eig_sys.solution,
libMesh::SystemNorm(libMeshEnums::L2));
if (abs(norms[i] - 1.0) > TEST_TOL) {
return 1;
}
}
const unsigned int dim = mesh.mesh_dimension();
const libMesh::DofMap & dof_map = eig_sys.get_dof_map();
libMesh::FEType fe_type = dof_map.variable_type(0);
libMesh::AutoPtr<libMesh::FEBase> fe(libMesh::FEBase::build(dim, fe_type));
libMesh::QGauss qrule(dim, libMeshEnums::FIFTH);
fe->attach_quadrature_rule(&qrule);
const std::vector<libMesh::Real> & JxW = fe->get_JxW();
const std::vector<std::vector<libMesh::Real> >& phi = fe->get_phi();
libMesh::AutoPtr<libMesh::NumericVector<libMesh::Real> > u, v;
double ui = 0.0;
double vj = 0.0;
double ip = 0.0;
for (i = 0; i < builder.num_req_eigenpairs - 1; i++) {
eig_sys.get_eigenpair(i);
u = eig_sys.solution->clone();
for (j = i + 1; j < builder.num_req_eigenpairs; j++) {
libMesh::MeshBase::const_element_iterator el = mesh.active_local_elements_begin();
libMesh::MeshBase::const_element_iterator end_el = mesh.active_local_elements_end();
eig_sys.get_eigenpair(j);
v = eig_sys.solution->clone();
for ( ; el != end_el; ++el) {
const libMesh::Elem * elem = *el;
fe->reinit(elem);
for (unsigned int qp = 0; qp < qrule.n_points(); qp++) {
for (unsigned int dof = 0; dof < phi.size(); dof++) {
ui += (*u)(dof) * phi[dof][qp];
vj += (*v)(dof) * phi[dof][qp];
}
ip += ui * vj * JxW[qp];
ui = 0.0;
vj = 0.0;
}
}
std::cerr << "INTEGRAL of " << i << " against " << j << " is: " << ip << std::endl;
if (abs(ip) > INTEGRATE_TOL) {
return 1;
}
ip = 0.0;
}
}
}
#endif // LIBMESH_HAVE_SLEPC
#ifdef QUESO_HAS_MPI
MPI_Finalize();
#endif
return 0;
#else
return 77;
#endif
}
void
InitialConditionTempl<T>::compute()
{
// -- NOTE ----
// The following code is a copy from libMesh project_vector.C plus it adds some features, so we
// can couple variable values
// and we also do not call any callbacks, but we use our initial condition system directly.
// ------------
// The dimension of the current element
_dim = _current_elem->dim();
// The element type
const ElemType elem_type = _current_elem->type();
// The number of nodes on the new element
const unsigned int n_nodes = _current_elem->n_nodes();
// Get FE objects of the appropriate type
// We cannot use the FE object in Assembly, since the following code is messing with the
// quadrature rules
// for projections and would screw it up. However, if we implement projections from one mesh to
// another,
// this code should use that implementation.
std::unique_ptr<FEBaseType> fe(FEBaseType::build(_dim, _fe_type));
// Prepare variables for projection
std::unique_ptr<QBase> qrule(_fe_type.default_quadrature_rule(_dim));
std::unique_ptr<QBase> qedgerule(_fe_type.default_quadrature_rule(1));
std::unique_ptr<QBase> qsiderule(_fe_type.default_quadrature_rule(_dim - 1));
// The values of the shape functions at the quadrature points
_phi = &fe->get_phi();
// The gradients of the shape functions at the quadrature points on the child element.
_dphi = nullptr;
_cont = fe->get_continuity();
if (_cont == C_ONE)
{
const std::vector<std::vector<GradientType>> & ref_dphi = fe->get_dphi();
_dphi = &ref_dphi;
}
// The Jacobian * quadrature weight at the quadrature points
_JxW = &fe->get_JxW();
// The XYZ locations of the quadrature points
_xyz_values = &fe->get_xyz();
// Update the DOF indices for this element based on the current mesh
_var.prepareIC();
_dof_indices = _var.dofIndices();
// The number of DOFs on the element
const unsigned int n_dofs = _dof_indices.size();
if (n_dofs == 0)
return;
// Fixed vs. free DoFs on edge/face projections
_dof_is_fixed.clear();
_dof_is_fixed.resize(n_dofs, false);
_free_dof.clear();
_free_dof.resize(n_dofs, 0);
// Zero the interpolated values
_Ue.resize(n_dofs);
_Ue.zero();
// In general, we need a series of
// projections to ensure a unique and continuous
// solution. We start by interpolating nodes, then
// hold those fixed and project edges, then
// hold those fixed and project faces, then
// hold those fixed and project interiors
// Interpolate node values first
_current_dof = 0;
for (_n = 0; _n != n_nodes; ++_n)
{
// FIXME: this should go through the DofMap,
// not duplicate _dof_indices code badly!
_nc = FEInterface::n_dofs_at_node(_dim, _fe_type, elem_type, _n);
if (!_current_elem->is_vertex(_n))
{
_current_dof += _nc;
continue;
}
if (_cont == DISCONTINUOUS)
libmesh_assert(_nc == 0);
else if (_cont == C_ZERO)
setCZeroVertices();
else if (_fe_type.family == HERMITE)
setHermiteVertices();
else if (_cont == C_ONE)
setOtherCOneVertices();
else
libmesh_error();
} // loop over nodes
// From here on out we won't be sampling at nodes anymore
_current_node = nullptr;
// In 3D, project any edge values next
if (_dim > 2 && _cont != DISCONTINUOUS)
for (unsigned int e = 0; e != _current_elem->n_edges(); ++e)
{
FEInterface::dofs_on_edge(_current_elem, _dim, _fe_type, e, _side_dofs);
// Some edge dofs are on nodes and already
// fixed, others are free to calculate
_free_dofs = 0;
for (unsigned int i = 0; i != _side_dofs.size(); ++i)
if (!_dof_is_fixed[_side_dofs[i]])
_free_dof[_free_dofs++] = i;
// There may be nothing to project
if (!_free_dofs)
continue;
// Initialize FE data on the edge
fe->attach_quadrature_rule(qedgerule.get());
fe->edge_reinit(_current_elem, e);
_n_qp = qedgerule->n_points();
choleskySolve(false);
}
// Project any side values (edges in 2D, faces in 3D)
if (_dim > 1 && _cont != DISCONTINUOUS)
for (unsigned int s = 0; s != _current_elem->n_sides(); ++s)
{
FEInterface::dofs_on_side(_current_elem, _dim, _fe_type, s, _side_dofs);
// Some side dofs are on nodes/edges and already
// fixed, others are free to calculate
_free_dofs = 0;
for (unsigned int i = 0; i != _side_dofs.size(); ++i)
if (!_dof_is_fixed[_side_dofs[i]])
_free_dof[_free_dofs++] = i;
// There may be nothing to project
if (!_free_dofs)
continue;
// Initialize FE data on the side
fe->attach_quadrature_rule(qsiderule.get());
fe->reinit(_current_elem, s);
_n_qp = qsiderule->n_points();
choleskySolve(false);
}
// Project the interior values, finally
// Some interior dofs are on nodes/edges/sides and
// already fixed, others are free to calculate
_free_dofs = 0;
for (unsigned int i = 0; i != n_dofs; ++i)
if (!_dof_is_fixed[i])
_free_dof[_free_dofs++] = i;
// There may be nothing to project
if (_free_dofs)
{
// Initialize FE data
fe->attach_quadrature_rule(qrule.get());
fe->reinit(_current_elem);
_n_qp = qrule->n_points();
choleskySolve(true);
} // if there are free interior dofs
// Make sure every DoF got reached!
for (unsigned int i = 0; i != n_dofs; ++i)
libmesh_assert(_dof_is_fixed[i]);
NumericVector<Number> & solution = _var.sys().solution();
// 'first' and 'last' are no longer used, see note about subdomain-restricted variables below
// const dof_id_type
// first = solution.first_local_index(),
// last = solution.last_local_index();
// Lock the new_vector since it is shared among threads.
{
Threads::spin_mutex::scoped_lock lock(Threads::spin_mtx);
for (unsigned int i = 0; i < n_dofs; i++)
// We may be projecting a new zero value onto
// an old nonzero approximation - RHS
// if (_Ue(i) != 0.)
// This is commented out because of subdomain restricted variables.
// It can be the case that if a subdomain restricted variable's boundary
// aligns perfectly with a processor boundary that the variable will get
// no value. To counteract this we're going to let every processor set a
// value at every node and then let PETSc figure it out.
// Later we can choose to do something different / better.
// if ((_dof_indices[i] >= first) && (_dof_indices[i] < last))
{
solution.set(_dof_indices[i], _Ue(i));
}
_var.setDofValues(_Ue);
}
}
void Biharmonic::JR::residual_and_jacobian(const NumericVector<Number> &u,
NumericVector<Number> *R,
SparseMatrix<Number> *J,
NonlinearImplicitSystem&)
{
#ifdef LIBMESH_ENABLE_SECOND_DERIVATIVES
if (!R && !J)
return;
// Declare a performance log. Give it a descriptive
// string to identify what part of the code we are
// logging, since there may be many PerfLogs in an
// application.
PerfLog perf_log ("Biharmonic Residual and Jacobian", false);
// A reference to the \p DofMap object for this system. The \p DofMap
// object handles the index translation from node and element numbers
// to degree of freedom numbers. We will talk more about the \p DofMap
// in future examples.
const DofMap& dof_map = get_dof_map();
// Get a constant reference to the Finite Element type
// for the first (and only) variable in the system.
FEType fe_type = dof_map.variable_type(0);
// Build a Finite Element object of the specified type. Since the
// \p FEBase::build() member dynamically creates memory we will
// store the object as an \p AutoPtr<FEBase>. This can be thought
// of as a pointer that will clean up after itself.
AutoPtr<FEBase> fe (FEBase::build(_biharmonic._dim, fe_type));
// Quadrature rule for numerical integration.
// With 2D triangles, the Clough quadrature rule puts a Gaussian
// quadrature rule on each of the 3 subelements
AutoPtr<QBase> qrule(fe_type.default_quadrature_rule(_biharmonic._dim));
// Tell the finite element object to use our quadrature rule.
fe->attach_quadrature_rule (qrule.get());
// Here we define some references to element-specific data that
// will be used to assemble the linear system.
// We begin with the element Jacobian * quadrature weight at each
// integration point.
const std::vector<Real>& JxW = fe->get_JxW();
// The element shape functions evaluated at the quadrature points.
const std::vector<std::vector<Real> >& phi = fe->get_phi();
// The element shape functions' derivatives evaluated at the quadrature points.
const std::vector<std::vector<RealGradient> >& dphi = fe->get_dphi();
// The element shape functions' second derivatives evaluated at the quadrature points.
const std::vector<std::vector<RealTensor> >& d2phi = fe->get_d2phi();
// For efficiency we will compute shape function laplacians n times,
// not n^2
std::vector<Real> Laplacian_phi_qp;
// Define data structures to contain the element matrix
// and right-hand-side vector contribution. Following
// basic finite element terminology we will denote these
// "Je" and "Re". More detail is in example 3.
DenseMatrix<Number> Je;
DenseVector<Number> Re;
// This vector will hold the degree of freedom indices for
// the element. These define where in the global system
// the element degrees of freedom get mapped.
std::vector<unsigned int> dof_indices;
// Old solution
const NumericVector<Number>& u_old = *old_local_solution;
// Now we will loop over all the elements in the mesh. We will
// compute the element matrix and right-hand-side contribution. See
// example 3 for a discussion of the element iterators.
MeshBase::const_element_iterator el = _biharmonic._mesh->active_local_elements_begin();
const MeshBase::const_element_iterator end_el = _biharmonic._mesh->active_local_elements_end();
for ( ; el != end_el; ++el) {
// Store a pointer to the element we are currently
// working on. This allows for nicer syntax later.
const Elem* elem = *el;
// Get the degree of freedom indices for the
// current element. These define where in the global
// matrix and right-hand-side this element will
// contribute to.
dof_map.dof_indices (elem, dof_indices);
// Compute the element-specific data for the current
// element. This involves computing the location of the
// quadrature points (q_point) and the shape function
// values/derivatives (phi, dphi,d2phi) for the current element.
fe->reinit (elem);
// Zero the element matrix, the right-hand side and the Laplacian matrix
// before summing them.
if (J)
Je.resize(dof_indices.size(), dof_indices.size());
if (R)
Re.resize(dof_indices.size());
Laplacian_phi_qp.resize(dof_indices.size());
for (unsigned int qp=0; qp<qrule->n_points(); qp++)
{
// AUXILIARY QUANTITIES:
// Residual and Jacobian share a few calculations:
// at the very least, in the case of interfacial energy only with a constant mobility,
// both calculations use Laplacian_phi_qp; more is shared the case of a concentration-dependent
// mobility and bulk potentials.
Number u_qp = 0.0, u_old_qp = 0.0, Laplacian_u_qp = 0.0, Laplacian_u_old_qp = 0.0;
Gradient grad_u_qp(0.0,0.0,0.0), grad_u_old_qp(0.0,0.0,0.0);
Number M_qp = 1.0, M_old_qp = 1.0, M_prime_qp = 0.0, M_prime_old_qp = 0.0;
for (unsigned int i=0; i<phi.size(); i++)
{
Laplacian_phi_qp[i] = d2phi[i][qp](0,0);
grad_u_qp(0) += u(dof_indices[i])*dphi[i][qp](0);
grad_u_old_qp(0) += u_old(dof_indices[i])*dphi[i][qp](0);
if (_biharmonic._dim > 1)
{
Laplacian_phi_qp[i] += d2phi[i][qp](1,1);
grad_u_qp(1) += u(dof_indices[i])*dphi[i][qp](1);
grad_u_old_qp(1) += u_old(dof_indices[i])*dphi[i][qp](1);
}
if (_biharmonic._dim > 2)
{
Laplacian_phi_qp[i] += d2phi[i][qp](2,2);
grad_u_qp(2) += u(dof_indices[i])*dphi[i][qp](2);
grad_u_old_qp(2) += u_old(dof_indices[i])*dphi[i][qp](2);
}
u_qp += phi[i][qp]*u(dof_indices[i]);
u_old_qp += phi[i][qp]*u_old(dof_indices[i]);
Laplacian_u_qp += Laplacian_phi_qp[i]*u(dof_indices[i]);
Laplacian_u_old_qp += Laplacian_phi_qp[i]*u_old(dof_indices[i]);
} // for i
if (_biharmonic._degenerate)
{
M_qp = 1.0 - u_qp*u_qp;
M_old_qp = 1.0 - u_old_qp*u_old_qp;
M_prime_qp = -2.0*u_qp;
M_prime_old_qp = -2.0*u_old_qp;
}
// ELEMENT RESIDUAL AND JACOBIAN
for (unsigned int i=0; i<phi.size(); i++)
{
// RESIDUAL
if (R)
{
Number ri = 0.0, ri_old = 0.0;
ri -= Laplacian_phi_qp[i]*M_qp*_biharmonic._kappa*Laplacian_u_qp;
ri_old -= Laplacian_phi_qp[i]*M_old_qp*_biharmonic._kappa*Laplacian_u_old_qp;
if (_biharmonic._degenerate)
{
ri -= (dphi[i][qp]*grad_u_qp)*M_prime_qp*(_biharmonic._kappa*Laplacian_u_qp);
ri_old -= (dphi[i][qp]*grad_u_old_qp)*M_prime_old_qp*(_biharmonic._kappa*Laplacian_u_old_qp);
}
if (_biharmonic._cahn_hillard)
{
if (_biharmonic._energy == DOUBLE_WELL || _biharmonic._energy == LOG_DOUBLE_WELL)
{
ri += Laplacian_phi_qp[i]*M_qp*_biharmonic._theta_c*(u_qp*u_qp - 1.0)*u_qp;
ri_old += Laplacian_phi_qp[i]*M_old_qp*_biharmonic._theta_c*(u_old_qp*u_old_qp - 1.0)*u_old_qp;
if (_biharmonic._degenerate)
{
ri += (dphi[i][qp]*grad_u_qp)*M_prime_qp*_biharmonic._theta_c*(u_qp*u_qp - 1.0)*u_qp;
ri_old += (dphi[i][qp]*grad_u_old_qp)*M_prime_old_qp*_biharmonic._theta_c*(u_old_qp*u_old_qp - 1.0)*u_old_qp;
}
}// if(_biharmonic._energy == DOUBLE_WELL || _biharmonic._energy == LOG_DOUBLE_WELL)
if (_biharmonic._energy == DOUBLE_OBSTACLE || _biharmonic._energy == LOG_DOUBLE_OBSTACLE)
{
ri -= Laplacian_phi_qp[i]*M_qp*_biharmonic._theta_c*u_qp;
ri_old -= Laplacian_phi_qp[i]*M_old_qp*_biharmonic._theta_c*u_old_qp;
if (_biharmonic._degenerate)
{
ri -= (dphi[i][qp]*grad_u_qp)*M_prime_qp*_biharmonic._theta_c*u_qp;
ri_old -= (dphi[i][qp]*grad_u_old_qp)*M_prime_old_qp*_biharmonic._theta_c*u_old_qp;
}
} // if(_biharmonic._energy == DOUBLE_OBSTACLE || _biharmonic._energy == LOG_DOUBLE_OBSTACLE)
if (_biharmonic._energy == LOG_DOUBLE_WELL || _biharmonic._energy == LOG_DOUBLE_OBSTACLE)
{
switch(_biharmonic._log_truncation)
{
case 2:
break;
case 3:
break;
default:
break;
}// switch(_biharmonic._log_truncation)
}// if(_biharmonic._energy == LOG_DOUBLE_WELL || _biharmonic._energy == LOG_DOUBLE_OBSTACLE)
}// if(_biharmonic._cahn_hillard)
Re(i) += JxW[qp]*((u_qp-u_old_qp)*phi[i][qp]-_biharmonic._dt*0.5*((2.0-_biharmonic._cnWeight)*ri + _biharmonic._cnWeight*ri_old));
} // if (R)
// JACOBIAN
if (J)
{
Number M_prime_prime_qp = 0.0;
if(_biharmonic._degenerate) M_prime_prime_qp = -2.0;
for (unsigned int j=0; j<phi.size(); j++)
{
Number ri_j = 0.0;
ri_j -= Laplacian_phi_qp[i]*M_qp*_biharmonic._kappa*Laplacian_phi_qp[j];
if (_biharmonic._degenerate)
{
ri_j -=
Laplacian_phi_qp[i]*M_prime_qp*phi[j][qp]*_biharmonic._kappa*Laplacian_u_qp +
(dphi[i][qp]*dphi[j][qp])*M_prime_qp*(_biharmonic._kappa*Laplacian_u_qp) +
(dphi[i][qp]*grad_u_qp)*(M_prime_prime_qp*phi[j][qp])*(_biharmonic._kappa*Laplacian_u_qp) +
(dphi[i][qp]*grad_u_qp)*(M_prime_qp)*(_biharmonic._kappa*Laplacian_phi_qp[j]);
}
if (_biharmonic._cahn_hillard)
{
if(_biharmonic._energy == DOUBLE_WELL || _biharmonic._energy == LOG_DOUBLE_WELL)
{
ri_j +=
Laplacian_phi_qp[i]*M_prime_qp*phi[j][qp]*_biharmonic._theta_c*(u_qp*u_qp - 1.0)*u_qp +
Laplacian_phi_qp[i]*M_qp*_biharmonic._theta_c*(3.0*u_qp*u_qp - 1.0)*phi[j][qp] +
(dphi[i][qp]*dphi[j][qp])*M_prime_qp*_biharmonic._theta_c*(u_qp*u_qp - 1.0)*u_qp +
(dphi[i][qp]*grad_u_qp)*M_prime_prime_qp*_biharmonic._theta_c*(u_qp*u_qp - 1.0)*u_qp +
(dphi[i][qp]*grad_u_qp)*M_prime_qp*_biharmonic._theta_c*(3.0*u_qp*u_qp - 1.0)*phi[j][qp];
}// if(_biharmonic._energy == DOUBLE_WELL || _biharmonic._energy == LOG_DOUBLE_WELL)
if (_biharmonic._energy == DOUBLE_OBSTACLE || _biharmonic._energy == LOG_DOUBLE_OBSTACLE)
{
ri_j -=
Laplacian_phi_qp[i]*M_prime_qp*phi[j][qp]*_biharmonic._theta_c*u_qp +
Laplacian_phi_qp[i]*M_qp*_biharmonic._theta_c*phi[j][qp] +
(dphi[i][qp]*dphi[j][qp])*M_prime_qp*_biharmonic._theta_c*u_qp +
(dphi[i][qp]*grad_u_qp)*M_prime_prime_qp*_biharmonic._theta_c*u_qp +
(dphi[i][qp]*grad_u_qp)*M_prime_qp*_biharmonic._theta_c*phi[j][qp];
} // if(_biharmonic._energy == DOUBLE_OBSTACLE || _biharmonic._energy == LOG_DOUBLE_OBSTACLE)
if (_biharmonic._energy == LOG_DOUBLE_WELL || _biharmonic._energy == LOG_DOUBLE_OBSTACLE)
{
switch(_biharmonic._log_truncation)
{
case 2:
break;
case 3:
break;
default:
break;
}// switch(_biharmonic._log_truncation)
}// if(_biharmonic._energy == LOG_DOUBLE_WELL || _biharmonic._energy == LOG_DOUBLE_OBSTACLE)
}// if(_biharmonic._cahn_hillard)
Je(i,j) += JxW[qp]*(phi[i][qp]*phi[j][qp] - 0.5*_biharmonic._dt*(2.0-_biharmonic._cnWeight)*ri_j);
} // for j
} // if (J)
} // for i
} // for qp
// The element matrix and right-hand-side are now built
// for this element. Add them to the global matrix and
// right-hand-side vector. The \p SparseMatrix::add_matrix()
// and \p NumericVector::add_vector() members do this for us.
// Start logging the insertion of the local (element)
// matrix and vector into the global matrix and vector
if (R)
{
// If the mesh has hanging nodes (e.g., as a result of refinement), those need to be constrained.
dof_map.constrain_element_vector(Re, dof_indices);
R->add_vector(Re, dof_indices);
}
if (J)
{
// If the mesh has hanging nodes (e.g., as a result of refinement), those need to be constrained.
dof_map.constrain_element_matrix(Je, dof_indices);
J->add_matrix(Je, dof_indices);
}
} // for el
#endif // LIBMESH_ENABLE_SECOND_DERIVATIVES
}
