int main(int argc, char *argv[])
{
// 1. Initialize MPI.
int num_procs, myid;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// 2. Parse command-line options.
const char *mesh_file = "../../data/star.mesh";
int order = 1;
bool set_bc = true;
bool static_cond = false;
bool hybridization = false;
bool visualization = 1;
bool use_petsc = true;
const char *petscrc_file = "";
bool use_nonoverlapping = false;
OptionsParser args(argc, argv);
args.AddOption(&mesh_file, "-m", "--mesh",
"Mesh file to use.");
args.AddOption(&order, "-o", "--order",
"Finite element order (polynomial degree).");
args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
"Impose or not essential boundary conditions.");
args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
" solution.");
args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
"--no-static-condensation", "Enable static condensation.");
args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
"--no-hybridization", "Enable hybridization.");
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
"Enable or disable GLVis visualization.");
args.AddOption(&use_petsc, "-usepetsc", "--usepetsc", "-no-petsc",
"--no-petsc",
"Use or not PETSc to solve the linear system.");
args.AddOption(&petscrc_file, "-petscopts", "--petscopts",
"PetscOptions file to use.");
args.AddOption(&use_nonoverlapping, "-nonoverlapping", "--nonoverlapping",
"-no-nonoverlapping", "--no-nonoverlapping",
"Use or not the block diagonal PETSc's matrix format "
"for non-overlapping domain decomposition.");
args.Parse();
if (!args.Good())
{
if (myid == 0)
{
args.PrintUsage(cout);
}
MPI_Finalize();
return 1;
}
if (myid == 0)
{
args.PrintOptions(cout);
}
// 2b. We initialize PETSc
if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
kappa = freq * M_PI;
// 3. Read the (serial) mesh from the given mesh file on all processors. We
// can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
// and volume, as well as periodic meshes with the same code.
Mesh *mesh = new Mesh(mesh_file, 1, 1);
int dim = mesh->Dimension();
int sdim = mesh->SpaceDimension();
// 4. Refine the serial mesh on all processors to increase the resolution. In
// this example we do 'ref_levels' of uniform refinement. We choose
// 'ref_levels' to be the largest number that gives a final mesh with no
// more than 1,000 elements.
{
int ref_levels =
(int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
for (int l = 0; l < ref_levels; l++)
{
mesh->UniformRefinement();
}
}
// 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
// this mesh further in parallel to increase the resolution. Once the
// parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
// meshes need to be reoriented before we can define high-order Nedelec
// spaces on them (this is needed in the ADS solver below).
ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
delete mesh;
{
int par_ref_levels = 2;
for (int l = 0; l < par_ref_levels; l++)
{
pmesh->UniformRefinement();
}
}
pmesh->ReorientTetMesh();
// 6. Define a parallel finite element space on the parallel mesh. Here we
// use the Raviart-Thomas finite elements of the specified order.
FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
HYPRE_Int size = fespace->GlobalTrueVSize();
if (myid == 0)
{
cout << "Number of finite element unknowns: " << size << endl;
}
// 7. Determine the list of true (i.e. parallel conforming) essential
// boundary dofs. In this example, the boundary conditions are defined
// by marking all the boundary attributes from the mesh as essential
// (Dirichlet) and converting them to a list of true dofs.
Array<int> ess_tdof_list;
if (pmesh->bdr_attributes.Size())
{
Array<int> ess_bdr(pmesh->bdr_attributes.Max());
ess_bdr = set_bc ? 1 : 0;
fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
}
// 8. Set up the parallel linear form b(.) which corresponds to the
// right-hand side of the FEM linear system, which in this case is
// (f,phi_i) where f is given by the function f_exact and phi_i are the
// basis functions in the finite element fespace.
VectorFunctionCoefficient f(sdim, f_exact);
ParLinearForm *b = new ParLinearForm(fespace);
b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
b->Assemble();
// 9. Define the solution vector x as a parallel finite element grid function
// corresponding to fespace. Initialize x by projecting the exact
// solution. Note that only values from the boundary faces will be used
// when eliminating the non-homogeneous boundary condition to modify the
// r.h.s. vector b.
ParGridFunction x(fespace);
VectorFunctionCoefficient F(sdim, F_exact);
x.ProjectCoefficient(F);
// 10. Set up the parallel bilinear form corresponding to the H(div)
// diffusion operator grad alpha div + beta I, by adding the div-div and
// the mass domain integrators.
Coefficient *alpha = new ConstantCoefficient(1.0);
Coefficient *beta = new ConstantCoefficient(1.0);
ParBilinearForm *a = new ParBilinearForm(fespace);
a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
// 11. Assemble the parallel bilinear form and the corresponding linear
// system, applying any necessary transformations such as: parallel
// assembly, eliminating boundary conditions, applying conforming
// constraints for non-conforming AMR, static condensation,
// hybridization, etc.
FiniteElementCollection *hfec = NULL;
ParFiniteElementSpace *hfes = NULL;
if (static_cond)
{
a->EnableStaticCondensation();
}
else if (hybridization)
{
hfec = new DG_Interface_FECollection(order-1, dim);
hfes = new ParFiniteElementSpace(pmesh, hfec);
a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
ess_tdof_list);
}
a->Assemble();
Vector B, X;
CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
pcg->SetRelTol(1e-12);
pcg->SetMaxIter(500);
pcg->SetPrintLevel(1);
if (!use_petsc)
{
HypreParMatrix A;
a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
HYPRE_Int glob_size = A.GetGlobalNumRows();
if (myid == 0)
{
cout << "Size of linear system: " << glob_size << endl;
}
// 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
// the 3D ADS preconditioners from hypre. If using hybridization, the
// system is preconditioned with hypre's BoomerAMG.
HypreSolver *prec = NULL;
pcg->SetOperator(A);
if (hybridization) { prec = new HypreBoomerAMG(A); }
else
{
ParFiniteElementSpace *prec_fespace =
(a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
if (dim == 2) { prec = new HypreAMS(A, prec_fespace); }
else { prec = new HypreADS(A, prec_fespace); }
}
pcg->SetPreconditioner(*prec);
pcg->Mult(B, X);
delete prec;
}
else
{
PetscParMatrix A;
PetscPreconditioner *prec = NULL;
a->SetOperatorType(use_nonoverlapping ?
Operator::PETSC_MATIS : Operator::PETSC_MATAIJ);
a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
if (myid == 0)
{
cout << "Size of linear system: " << A.M() << endl;
}
pcg->SetOperator(A);
if (use_nonoverlapping)
{
ParFiniteElementSpace *prec_fespace =
(a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
// Auxiliary class for BDDC customization
PetscBDDCSolverParams opts;
// Inform the solver about the finite element space
opts.SetSpace(prec_fespace);
// Inform the solver about essential dofs
opts.SetEssBdrDofs(&ess_tdof_list);
// Create a BDDC solver with parameters
prec = new PetscBDDCSolver(A, opts);
}
else
{
// Create an empty preconditioner that can be customized at runtime.
prec = new PetscPreconditioner(A, "solver_");
}
pcg->SetPreconditioner(*prec);
pcg->Mult(B, X);
delete prec;
}
delete pcg;
// 13. Recover the parallel grid function corresponding to X. This is the
// local finite element solution on each processor.
a->RecoverFEMSolution(X, *b, x);
// 14. Compute and print the L^2 norm of the error.
{
double err = x.ComputeL2Error(F);
if (myid == 0)
{
cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
}
}
// 15. Save the refined mesh and the solution in parallel. This output can
// be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
{
ostringstream mesh_name, sol_name;
mesh_name << "mesh." << setfill('0') << setw(6) << myid;
sol_name << "sol." << setfill('0') << setw(6) << myid;
ofstream mesh_ofs(mesh_name.str().c_str());
mesh_ofs.precision(8);
pmesh->Print(mesh_ofs);
ofstream sol_ofs(sol_name.str().c_str());
sol_ofs.precision(8);
x.Save(sol_ofs);
}
// 16. Send the solution by socket to a GLVis server.
if (visualization)
{
char vishost[] = "localhost";
int visport = 19916;
socketstream sol_sock(vishost, visport);
sol_sock << "parallel " << num_procs << " " << myid << "\n";
sol_sock.precision(8);
sol_sock << "solution\n" << *pmesh << x << flush;
}
// 17. Free the used memory.
delete hfes;
delete hfec;
delete a;
delete alpha;
delete beta;
delete b;
delete fespace;
delete fec;
delete pmesh;
// We finalize PETSc
if (use_petsc) { MFEMFinalizePetsc(); }
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[])
{
// 1. Initialize MPI.
int num_procs, myid;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// 2. Parse command-line options.
const char *mesh_file = "../data/beam-tet.mesh";
int order = 1;
bool visualization = 1;
OptionsParser args(argc, argv);
args.AddOption(&mesh_file, "-m", "--mesh",
"Mesh file to use.");
args.AddOption(&order, "-o", "--order",
"Finite element order (polynomial degree).");
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
"Enable or disable GLVis visualization.");
args.Parse();
if (!args.Good())
{
if (myid == 0)
args.PrintUsage(cout);
MPI_Finalize();
return 1;
}
if (myid == 0)
args.PrintOptions(cout);
// 3. Read the (serial) mesh from the given mesh file on all processors. We
// can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
// and volume meshes with the same code.
Mesh *mesh;
ifstream imesh(mesh_file);
if (!imesh)
{
if (myid == 0)
cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
MPI_Finalize();
return 2;
}
mesh = new Mesh(imesh, 1, 1);
imesh.close();
int dim = mesh->Dimension();
if (dim != 3)
{
if (myid == 0)
cerr << "\nThis example requires a 3D mesh\n" << endl;
MPI_Finalize();
return 3;
}
// 4. Refine the serial mesh on all processors to increase the resolution. In
// this example we do 'ref_levels' of uniform refinement. We choose
// 'ref_levels' to be the largest number that gives a final mesh with no
// more than 1,000 elements.
{
int ref_levels =
(int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
for (int l = 0; l < ref_levels; l++)
mesh->UniformRefinement();
}
// 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
// this mesh further in parallel to increase the resolution. Once the
// parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
// meshes need to be reoriented before we can define high-order Nedelec
// spaces on them.
ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
delete mesh;
{
int par_ref_levels = 2;
for (int l = 0; l < par_ref_levels; l++)
pmesh->UniformRefinement();
}
pmesh->ReorientTetMesh();
// 6. Define a parallel finite element space on the parallel mesh. Here we
// use the lowest order Nedelec finite elements, but we can easily switch
// to higher-order spaces by changing the value of p.
FiniteElementCollection *fec = new ND_FECollection(order, dim);
ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
int size = fespace->GlobalTrueVSize();
if (myid == 0)
cout << "Number of unknowns: " << size << endl;
// 7. Set up the parallel linear form b(.) which corresponds to the
// right-hand side of the FEM linear system, which in this case is
// (f,phi_i) where f is given by the function f_exact and phi_i are the
// basis functions in the finite element fespace.
VectorFunctionCoefficient f(3, f_exact);
ParLinearForm *b = new ParLinearForm(fespace);
b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
b->Assemble();
// 8. Define the solution vector x as a parallel finite element grid function
// corresponding to fespace. Initialize x by projecting the exact
// solution. Note that only values from the boundary edges will be used
// when eliminating the non-homogeneous boundary condition to modify the
// r.h.s. vector b.
ParGridFunction x(fespace);
VectorFunctionCoefficient E(3, E_exact);
x.ProjectCoefficient(E);
// 9. Set up the parallel bilinear form corresponding to the EM diffusion
// operator curl muinv curl + sigma I, by adding the curl-curl and the
// mass domain integrators and finally imposing non-homogeneous Dirichlet
// boundary conditions. The boundary conditions are implemented by
// marking all the boundary attributes from the mesh as essential
// (Dirichlet). After serial and parallel assembly we extract the
// parallel matrix A.
Coefficient *muinv = new ConstantCoefficient(1.0);
Coefficient *sigma = new ConstantCoefficient(1.0);
ParBilinearForm *a = new ParBilinearForm(fespace);
a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
a->Assemble();
Array<int> ess_bdr(pmesh->bdr_attributes.Max());
ess_bdr = 1;
a->EliminateEssentialBC(ess_bdr, x, *b);
a->Finalize();
// 10. Define the parallel (hypre) matrix and vectors representing a(.,.),
// b(.) and the finite element approximation.
HypreParMatrix *A = a->ParallelAssemble();
HypreParVector *B = b->ParallelAssemble();
HypreParVector *X = x.ParallelAverage();
*X = 0.0;
delete a;
delete sigma;
delete muinv;
delete b;
// 11. Define and apply a parallel PCG solver for AX=B with the AMS
// preconditioner from hypre.
HypreSolver *ams = new HypreAMS(*A, fespace);
HyprePCG *pcg = new HyprePCG(*A);
pcg->SetTol(1e-12);
pcg->SetMaxIter(500);
pcg->SetPrintLevel(2);
pcg->SetPreconditioner(*ams);
pcg->Mult(*B, *X);
// 12. Extract the parallel grid function corresponding to the finite element
// approximation X. This is the local solution on each processor.
x = *X;
// 13. Compute and print the L^2 norm of the error.
{
double err = x.ComputeL2Error(E);
if (myid == 0)
cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
}
// 14. Save the refined mesh and the solution in parallel. This output can
// be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
{
ostringstream mesh_name, sol_name;
mesh_name << "mesh." << setfill('0') << setw(6) << myid;
sol_name << "sol." << setfill('0') << setw(6) << myid;
ofstream mesh_ofs(mesh_name.str().c_str());
mesh_ofs.precision(8);
pmesh->Print(mesh_ofs);
ofstream sol_ofs(sol_name.str().c_str());
sol_ofs.precision(8);
x.Save(sol_ofs);
}
// 15. Send the solution by socket to a GLVis server.
if (visualization)
{
char vishost[] = "localhost";
int visport = 19916;
socketstream sol_sock(vishost, visport);
sol_sock << "parallel " << num_procs << " " << myid << "\n";
sol_sock.precision(8);
sol_sock << "solution\n" << *pmesh << x << flush;
}
// 16. Free the used memory.
delete pcg;
delete ams;
delete X;
delete B;
delete A;
delete fespace;
delete fec;
delete pmesh;
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[])
{
// 1. Initialize MPI.
int num_procs, myid;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// 2. Parse command-line options.
int elem_type = 1;
int ref_levels = 2;
int amr = 0;
int order = 2;
bool always_snap = false;
bool visualization = 1;
OptionsParser args(argc, argv);
args.AddOption(&elem_type, "-e", "--elem",
"Type of elements to use: 0 - triangles, 1 - quads.");
args.AddOption(&order, "-o", "--order",
"Finite element order (polynomial degree).");
args.AddOption(&ref_levels, "-r", "--refine",
"Number of times to refine the mesh uniformly.");
args.AddOption(&amr, "-amr", "--refine-locally",
"Additional local (non-conforming) refinement:"
" 1 = refine around north pole, 2 = refine randomly.");
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
"Enable or disable GLVis visualization.");
args.AddOption(&always_snap, "-snap", "--always-snap", "-no-snap",
"--snap-at-the-end",
"If true, snap nodes to the sphere initially and after each refinement "
"otherwise, snap only after the last refinement");
args.Parse();
if (!args.Good())
{
if (myid == 0)
{
args.PrintUsage(cout);
}
MPI_Finalize();
return 1;
}
if (myid == 0)
{
args.PrintOptions(cout);
}
// 3. Generate an initial high-order (surface) mesh on the unit sphere. The
// Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
// the elements and the vertices of the mesh, and then make it high-order
// by specifying a finite element space for its nodes.
int Nvert = 8, Nelem = 6;
if (elem_type == 0)
{
Nvert = 6;
Nelem = 8;
}
Mesh *mesh = new Mesh(2, Nvert, Nelem, 0, 3);
if (elem_type == 0) // inscribed octahedron
{
const double tri_v[6][3] =
{
{ 1, 0, 0}, { 0, 1, 0}, {-1, 0, 0},
{ 0, -1, 0}, { 0, 0, 1}, { 0, 0, -1}
};
const int tri_e[8][3] =
{
{0, 1, 4}, {1, 2, 4}, {2, 3, 4}, {3, 0, 4},
{1, 0, 5}, {2, 1, 5}, {3, 2, 5}, {0, 3, 5}
};
for (int j = 0; j < Nvert; j++)
{
mesh->AddVertex(tri_v[j]);
}
for (int j = 0; j < Nelem; j++)
{
int attribute = j + 1;
mesh->AddTriangle(tri_e[j], attribute);
}
mesh->FinalizeTriMesh(1, 1, true);
}
else // inscribed cube
{
const double quad_v[8][3] =
{
{-1, -1, -1}, {+1, -1, -1}, {+1, +1, -1}, {-1, +1, -1},
{-1, -1, +1}, {+1, -1, +1}, {+1, +1, +1}, {-1, +1, +1}
};
const int quad_e[6][4] =
{
{3, 2, 1, 0}, {0, 1, 5, 4}, {1, 2, 6, 5},
{2, 3, 7, 6}, {3, 0, 4, 7}, {4, 5, 6, 7}
};
for (int j = 0; j < Nvert; j++)
{
mesh->AddVertex(quad_v[j]);
}
for (int j = 0; j < Nelem; j++)
{
int attribute = j + 1;
mesh->AddQuad(quad_e[j], attribute);
}
mesh->FinalizeQuadMesh(1, 1, true);
}
// Set the space for the high-order mesh nodes.
H1_FECollection fec(order, mesh->Dimension());
FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
mesh->SetNodalFESpace(&nodal_fes);
// 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
// refinements is fixed to 2.
for (int l = 0; l <= ref_levels; l++)
{
if (l > 0) // for l == 0 just perform snapping
{
mesh->UniformRefinement();
}
// Snap the nodes of the refined mesh back to sphere surface.
if (always_snap)
{
SnapNodes(*mesh);
}
}
if (amr == 1)
{
for (int l = 0; l < 3; l++)
{
mesh->RefineAtVertex(Vertex(0, 0, 1));
}
SnapNodes(*mesh);
}
else if (amr == 2)
{
for (int l = 0; l < 2; l++)
{
mesh->RandomRefinement(0.5); // 50% probability
}
SnapNodes(*mesh);
}
ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
delete mesh;
{
int par_ref_levels = 2;
for (int l = 0; l < par_ref_levels; l++)
{
pmesh->UniformRefinement();
// Snap the nodes of the refined mesh back to sphere surface.
if (always_snap)
{
SnapNodes(*pmesh);
}
}
if (!always_snap || par_ref_levels < 1)
{
SnapNodes(*pmesh);
}
}
if (amr == 1)
{
for (int l = 0; l < 2; l++)
{
pmesh->RefineAtVertex(Vertex(0, 0, 1));
}
SnapNodes(*pmesh);
}
else if (amr == 2)
{
for (int l = 0; l < 2; l++)
{
pmesh->RandomRefinement(0.5); // 50% probability
}
SnapNodes(*pmesh);
}
// 5. Define a finite element space on the mesh. Here we use isoparametric
// finite elements -- the same as the mesh nodes.
ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
HYPRE_Int size = fespace->GlobalTrueVSize();
if (myid == 0)
{
cout << "Number of unknowns: " << size << endl;
}
// 6. Set up the linear form b(.) which corresponds to the right-hand side of
// the FEM linear system, which in this case is (1,phi_i) where phi_i are
// the basis functions in the finite element fespace.
ParLinearForm *b = new ParLinearForm(fespace);
ConstantCoefficient one(1.0);
FunctionCoefficient rhs_coef (analytic_rhs);
FunctionCoefficient sol_coef (analytic_solution);
b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
b->Assemble();
// 7. Define the solution vector x as a finite element grid function
// corresponding to fespace. Initialize x with initial guess of zero.
ParGridFunction x(fespace);
x = 0.0;
// 8. Set up the bilinear form a(.,.) on the finite element space
// corresponding to the Laplacian operator -Delta, by adding the Diffusion
// and Mass domain integrators.
ParBilinearForm *a = new ParBilinearForm(fespace);
a->AddDomainIntegrator(new DiffusionIntegrator(one));
a->AddDomainIntegrator(new MassIntegrator(one));
// 9. Assemble the parallel linear system, applying any transformations
// such as: parallel assembly, applying conforming constraints, etc.
a->Assemble();
HypreParMatrix A;
Vector B, X;
Array<int> empty_tdof_list;
a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);
// 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
// preconditioner from hypre. Extract the parallel grid function x
// corresponding to the finite element approximation X. This is the local
// solution on each processor.
HypreSolver *amg = new HypreBoomerAMG(A);
HyprePCG *pcg = new HyprePCG(A);
pcg->SetTol(1e-12);
pcg->SetMaxIter(200);
pcg->SetPrintLevel(2);
pcg->SetPreconditioner(*amg);
pcg->Mult(B, X);
a->RecoverFEMSolution(X, *b, x);
delete a;
delete b;
// 11. Compute and print the L^2 norm of the error.
double err = x.ComputeL2Error(sol_coef);
if (myid == 0)
{
cout << "\nL2 norm of error: " << err << endl;
}
// 12. Save the refined mesh and the solution. This output can be viewed
// later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
{
ostringstream mesh_name, sol_name;
mesh_name << "sphere_refined." << setfill('0') << setw(6) << myid;
sol_name << "sol." << setfill('0') << setw(6) << myid;
ofstream mesh_ofs(mesh_name.str().c_str());
mesh_ofs.precision(8);
pmesh->Print(mesh_ofs);
ofstream sol_ofs(sol_name.str().c_str());
sol_ofs.precision(8);
x.Save(sol_ofs);
}
// 13. Send the solution by socket to a GLVis server.
if (visualization)
{
char vishost[] = "localhost";
int visport = 19916;
socketstream sol_sock(vishost, visport);
sol_sock << "parallel " << num_procs << " " << myid << "\n";
sol_sock.precision(8);
sol_sock << "solution\n" << *pmesh << x << flush;
}
// 14. Free the used memory.
delete pcg;
delete amg;
delete fespace;
delete pmesh;
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[])
{
// 1. Initialize MPI.
int num_procs, myid;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// 2. Parse command-line options.
const char *mesh_file = "../data/beam-tet.mesh";
int ser_ref_levels = 2;
int par_ref_levels = 1;
int order = 1;
int nev = 5;
bool visualization = 1;
OptionsParser args(argc, argv);
args.AddOption(&mesh_file, "-m", "--mesh",
"Mesh file to use.");
args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
"Number of times to refine the mesh uniformly in serial.");
args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
"Number of times to refine the mesh uniformly in parallel.");
args.AddOption(&order, "-o", "--order",
"Finite element order (polynomial degree) or -1 for"
" isoparametric space.");
args.AddOption(&nev, "-n", "--num-eigs",
"Number of desired eigenmodes.");
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
"Enable or disable GLVis visualization.");
args.Parse();
if (!args.Good())
{
if (myid == 0)
{
args.PrintUsage(cout);
}
MPI_Finalize();
return 1;
}
if (myid == 0)
{
args.PrintOptions(cout);
}
// 3. Read the (serial) mesh from the given mesh file on all processors. We
// can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
// and volume meshes with the same code.
Mesh *mesh;
ifstream imesh(mesh_file);
if (!imesh)
{
if (myid == 0)
{
cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
}
MPI_Finalize();
return 2;
}
mesh = new Mesh(imesh, 1, 1);
imesh.close();
int dim = mesh->Dimension();
// 4. Refine the serial mesh on all processors to increase the resolution. In
// this example we do 'ref_levels' of uniform refinement (2 by default, or
// specified on the command line with -rs).
for (int lev = 0; lev < ser_ref_levels; lev++)
{
mesh->UniformRefinement();
}
// 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
// this mesh further in parallel to increase the resolution (1 time by
// default, or specified on the command line with -rp). Once the parallel
// mesh is defined, the serial mesh can be deleted.
ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
delete mesh;
for (int lev = 0; lev < par_ref_levels; lev++)
{
pmesh->UniformRefinement();
}
// 6. Define a parallel finite element space on the parallel mesh. Here we
// use the Nedelec finite elements of the specified order.
FiniteElementCollection *fec = new ND_FECollection(order, dim);
ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
HYPRE_Int size = fespace->GlobalTrueVSize();
if (myid == 0)
{
cout << "Number of unknowns: " << size << endl;
}
// 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
// element space. The first corresponds to the curl curl, while the second
// is a simple mass matrix needed on the right hand side of the
// generalized eigenvalue problem below. The boundary conditions are
// implemented by marking all the boundary attributes from the mesh as
// essential. The corresponding degrees of freedom are eliminated with
// special values on the diagonal to shift the Dirichlet eigenvalues out
// of the computational range. After serial and parallel assembly we
// extract the corresponding parallel matrices A and M.
ConstantCoefficient one(1.0);
Array<int> ess_bdr;
if (pmesh->bdr_attributes.Size())
{
ess_bdr.SetSize(pmesh->bdr_attributes.Max());
ess_bdr = 1;
}
ParBilinearForm *a = new ParBilinearForm(fespace);
a->AddDomainIntegrator(new CurlCurlIntegrator(one));
if (pmesh->bdr_attributes.Size() == 0)
{
// Add a mass term if the mesh has no boundary, e.g. periodic mesh or
// closed surface.
a->AddDomainIntegrator(new VectorFEMassIntegrator(one));
}
a->Assemble();
a->EliminateEssentialBCDiag(ess_bdr, 1.0);
a->Finalize();
ParBilinearForm *m = new ParBilinearForm(fespace);
m->AddDomainIntegrator(new VectorFEMassIntegrator(one));
m->Assemble();
// shift the eigenvalue corresponding to eliminated dofs to a large value
m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
m->Finalize();
HypreParMatrix *A = a->ParallelAssemble();
HypreParMatrix *M = m->ParallelAssemble();
delete a;
delete m;
// 8. Define and configure the AME eigensolver and the AMS preconditioner for
// A to be used within the solver. Set the matrices which define the
// generalized eigenproblem A x = lambda M x.
HypreAMS *ams = new HypreAMS(*A,fespace);
ams->SetPrintLevel(0);
ams->SetSingularProblem();
HypreAME *ame = new HypreAME(MPI_COMM_WORLD);
ame->SetNumModes(nev);
ame->SetPreconditioner(*ams);
ame->SetMaxIter(100);
ame->SetTol(1e-8);
ame->SetPrintLevel(1);
ame->SetMassMatrix(*M);
ame->SetOperator(*A);
// 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
// parallel grid function to represent each of the eigenmodes returned by
// the solver.
Array<double> eigenvalues;
ame->Solve();
ame->GetEigenvalues(eigenvalues);
ParGridFunction x(fespace);
// 10. Save the refined mesh and the modes in parallel. This output can be
// viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
{
ostringstream mesh_name, mode_name;
mesh_name << "mesh." << setfill('0') << setw(6) << myid;
ofstream mesh_ofs(mesh_name.str().c_str());
mesh_ofs.precision(8);
pmesh->Print(mesh_ofs);
for (int i=0; i<nev; i++)
{
// convert eigenvector from HypreParVector to ParGridFunction
x = ame->GetEigenvector(i);
mode_name << "mode_" << setfill('0') << setw(2) << i << "."
<< setfill('0') << setw(6) << myid;
ofstream mode_ofs(mode_name.str().c_str());
mode_ofs.precision(8);
x.Save(mode_ofs);
mode_name.str("");
}
}
// 11. Send the solution by socket to a GLVis server.
if (visualization)
{
char vishost[] = "localhost";
int visport = 19916;
socketstream mode_sock(vishost, visport);
mode_sock.precision(8);
for (int i=0; i<nev; i++)
{
if ( myid == 0 )
{
cout << "Eigenmode " << i+1 << '/' << nev
<< ", Lambda = " << eigenvalues[i] << endl;
}
// convert eigenvector from HypreParVector to ParGridFunction
x = ame->GetEigenvector(i);
mode_sock << "parallel " << num_procs << " " << myid << "\n"
<< "solution\n" << *pmesh << x << flush
<< "window_title 'Eigenmode " << i+1 << '/' << nev
<< ", Lambda = " << eigenvalues[i] << "'" << endl;
char c;
if (myid == 0)
{
cout << "press (q)uit or (c)ontinue --> " << flush;
cin >> c;
}
MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
if (c != 'c')
{
break;
}
}
mode_sock.close();
}
double L2ZZErrorEstimator(BilinearFormIntegrator &flux_integrator,
const ParGridFunction &x,
ParFiniteElementSpace &smooth_flux_fes,
ParFiniteElementSpace &flux_fes,
Vector &errors,
int norm_p, double solver_tol, int solver_max_it)
{
// Compute fluxes in discontinuous space
GridFunction flux(&flux_fes);
flux = 0.0;
ParFiniteElementSpace *xfes = x.ParFESpace();
Array<int> xdofs, fdofs;
Vector el_x, el_f;
for (int i = 0; i < xfes->GetNE(); i++)
{
xfes->GetElementVDofs(i, xdofs);
x.GetSubVector(xdofs, el_x);
ElementTransformation *Transf = xfes->GetElementTransformation(i);
flux_integrator.ComputeElementFlux(*xfes->GetFE(i), *Transf, el_x,
*flux_fes.GetFE(i), el_f, false);
flux_fes.GetElementVDofs(i, fdofs);
flux.AddElementVector(fdofs, el_f);
}
// Assemble the linear system for L2 projection into the "smooth" space
ParBilinearForm *a = new ParBilinearForm(&smooth_flux_fes);
ParLinearForm *b = new ParLinearForm(&smooth_flux_fes);
VectorGridFunctionCoefficient f(&flux);
if (xfes->GetNE())
{
if (smooth_flux_fes.GetFE(0)->GetRangeType() == FiniteElement::SCALAR)
{
VectorMassIntegrator *vmass = new VectorMassIntegrator;
vmass->SetVDim(smooth_flux_fes.GetVDim());
a->AddDomainIntegrator(vmass);
b->AddDomainIntegrator(new VectorDomainLFIntegrator(f));
}
else
{
a->AddDomainIntegrator(new VectorFEMassIntegrator);
b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
}
}
b->Assemble();
a->Assemble();
a->Finalize();
// The destination of the projected discontinuous flux
ParGridFunction smooth_flux(&smooth_flux_fes);
smooth_flux = 0.0;
HypreParMatrix* A = a->ParallelAssemble();
HypreParVector* B = b->ParallelAssemble();
HypreParVector* X = smooth_flux.ParallelProject();
delete a;
delete b;
// Define and apply a parallel PCG solver for AX=B with the BoomerAMG
// preconditioner from hypre.
HypreBoomerAMG *amg = new HypreBoomerAMG(*A);
amg->SetPrintLevel(0);
HyprePCG *pcg = new HyprePCG(*A);
pcg->SetTol(solver_tol);
pcg->SetMaxIter(solver_max_it);
pcg->SetPrintLevel(0);
pcg->SetPreconditioner(*amg);
pcg->Mult(*B, *X);
// Extract the parallel grid function corresponding to the finite element
// approximation X. This is the local solution on each processor.
smooth_flux = *X;
delete A;
delete B;
delete X;
delete amg;
delete pcg;
// Proceed through the elements one by one, and find the Lp norm differences
// between the flux as computed per element and the flux projected onto the
// smooth_flux_fes space.
double total_error = 0.0;
errors.SetSize(xfes->GetNE());
for (int i = 0; i < xfes->GetNE(); i++)
{
errors(i) = ComputeElementLpDistance(norm_p, i, smooth_flux, flux);
total_error += pow(errors(i), norm_p);
}
double glob_error;
MPI_Allreduce(&total_error, &glob_error, 1, MPI_DOUBLE, MPI_SUM,
xfes->GetComm());
return pow(glob_error, 1.0/norm_p);
}
int main(int argc, char *argv[])
{
// 1. Initialize MPI.
int num_procs, myid;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// 2. Parse command-line options.
const char *mesh_file = "../data/beam-tri.mesh";
int order = 1;
int nev = 5;
bool visualization = 1;
bool amg_elast = 0;
OptionsParser args(argc, argv);
args.AddOption(&mesh_file, "-m", "--mesh",
"Mesh file to use.");
args.AddOption(&order, "-o", "--order",
"Finite element order (polynomial degree).");
args.AddOption(&nev, "-n", "--num-eigs",
"Number of desired eigenmodes.");
args.AddOption(&amg_elast, "-elast", "--amg-for-elasticity", "-sys",
"--amg-for-systems",
"Use the special AMG elasticity solver (GM/LN approaches), "
"or standard AMG for systems (unknown approach).");
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
"Enable or disable GLVis visualization.");
args.Parse();
if (!args.Good())
{
if (myid == 0)
{
args.PrintUsage(cout);
}
MPI_Finalize();
return 1;
}
if (myid == 0)
{
args.PrintOptions(cout);
}
// 3. Read the (serial) mesh from the given mesh file on all processors. We
// can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
// and volume meshes with the same code.
Mesh *mesh = new Mesh(mesh_file, 1, 1);
int dim = mesh->Dimension();
if (mesh->attributes.Max() < 2)
{
if (myid == 0)
cerr << "\nInput mesh should have at least two materials!"
<< " (See schematic in ex12p.cpp)\n"
<< endl;
MPI_Finalize();
return 3;
}
// 4. Select the order of the finite element discretization space. For NURBS
// meshes, we increase the order by degree elevation.
if (mesh->NURBSext && order > mesh->NURBSext->GetOrder())
{
mesh->DegreeElevate(order - mesh->NURBSext->GetOrder());
}
// 5. Refine the serial mesh on all processors to increase the resolution. In
// this example we do 'ref_levels' of uniform refinement. We choose
// 'ref_levels' to be the largest number that gives a final mesh with no
// more than 1,000 elements.
{
int ref_levels =
(int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
for (int l = 0; l < ref_levels; l++)
{
mesh->UniformRefinement();
}
}
// 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
// this mesh further in parallel to increase the resolution. Once the
// parallel mesh is defined, the serial mesh can be deleted.
ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
delete mesh;
{
int par_ref_levels = 1;
for (int l = 0; l < par_ref_levels; l++)
{
pmesh->UniformRefinement();
}
}
// 7. Define a parallel finite element space on the parallel mesh. Here we
// use vector finite elements, i.e. dim copies of a scalar finite element
// space. We use the ordering by vector dimension (the last argument of
// the FiniteElementSpace constructor) which is expected in the systems
// version of BoomerAMG preconditioner. For NURBS meshes, we use the
// (degree elevated) NURBS space associated with the mesh nodes.
FiniteElementCollection *fec;
ParFiniteElementSpace *fespace;
const bool use_nodal_fespace = pmesh->NURBSext && !amg_elast;
if (use_nodal_fespace)
{
fec = NULL;
fespace = (ParFiniteElementSpace *)pmesh->GetNodes()->FESpace();
}
else
{
fec = new H1_FECollection(order, dim);
fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
}
HYPRE_Int size = fespace->GlobalTrueVSize();
if (myid == 0)
{
cout << "Number of unknowns: " << size << endl
<< "Assembling: " << flush;
}
// 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
// element space corresponding to the linear elasticity integrator with
// piece-wise constants coefficient lambda and mu, a simple mass matrix
// needed on the right hand side of the generalized eigenvalue problem
// below. The boundary conditions are implemented by marking only boundary
// attribute 1 as essential. We use special values on the diagonal to
// shift the Dirichlet eigenvalues out of the computational range. After
// serial/parallel assembly we extract the corresponding parallel matrices
// A and M.
Vector lambda(pmesh->attributes.Max());
lambda = 1.0;
lambda(0) = lambda(1)*50;
PWConstCoefficient lambda_func(lambda);
Vector mu(pmesh->attributes.Max());
mu = 1.0;
mu(0) = mu(1)*50;
PWConstCoefficient mu_func(mu);
Array<int> ess_bdr(pmesh->bdr_attributes.Max());
ess_bdr = 0;
ess_bdr[0] = 1;
ParBilinearForm *a = new ParBilinearForm(fespace);
a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
if (myid == 0)
{
cout << "matrix ... " << flush;
}
a->Assemble();
a->EliminateEssentialBCDiag(ess_bdr, 1.0);
a->Finalize();
ParBilinearForm *m = new ParBilinearForm(fespace);
m->AddDomainIntegrator(new VectorMassIntegrator());
m->Assemble();
// shift the eigenvalue corresponding to eliminated dofs to a large value
m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
m->Finalize();
if (myid == 0)
{
cout << "done." << endl;
}
HypreParMatrix *A = a->ParallelAssemble();
HypreParMatrix *M = m->ParallelAssemble();
delete a;
delete m;
// 9. Define and configure the LOBPCG eigensolver and the BoomerAMG
// preconditioner for A to be used within the solver. Set the matrices
// which define the generalized eigenproblem A x = lambda M x.
HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
amg->SetPrintLevel(0);
if (amg_elast)
{
amg->SetElasticityOptions(fespace);
}
else
{
amg->SetSystemsOptions(dim);
}
HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
lobpcg->SetNumModes(nev);
lobpcg->SetPreconditioner(*amg);
lobpcg->SetMaxIter(100);
lobpcg->SetTol(1e-8);
lobpcg->SetPrecondUsageMode(1);
lobpcg->SetPrintLevel(1);
lobpcg->SetMassMatrix(*M);
lobpcg->SetOperator(*A);
// 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
// parallel grid function to represent each of the eigenmodes returned by
// the solver.
Array<double> eigenvalues;
lobpcg->Solve();
lobpcg->GetEigenvalues(eigenvalues);
ParGridFunction x(fespace);
// 11. For non-NURBS meshes, make the mesh curved based on the finite element
// space. This means that we define the mesh elements through a fespace
// based transformation of the reference element. This allows us to save
// the displaced mesh as a curved mesh when using high-order finite
// element displacement field. We assume that the initial mesh (read from
// the file) is not higher order curved mesh compared to the chosen FE
// space.
if (!use_nodal_fespace)
{
pmesh->SetNodalFESpace(fespace);
}
// 12. Save the refined mesh and the modes in parallel. This output can be
// viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
{
ostringstream mesh_name, mode_name;
mesh_name << "mesh." << setfill('0') << setw(6) << myid;
ofstream mesh_ofs(mesh_name.str().c_str());
mesh_ofs.precision(8);
pmesh->Print(mesh_ofs);
for (int i=0; i<nev; i++)
{
// convert eigenvector from HypreParVector to ParGridFunction
x = lobpcg->GetEigenvector(i);
mode_name << "mode_" << setfill('0') << setw(2) << i << "."
<< setfill('0') << setw(6) << myid;
ofstream mode_ofs(mode_name.str().c_str());
mode_ofs.precision(8);
x.Save(mode_ofs);
mode_name.str("");
}
}
// 13. Send the above data by socket to a GLVis server. Use the "n" and "b"
// keys in GLVis to visualize the displacements.
if (visualization)
{
char vishost[] = "localhost";
int visport = 19916;
socketstream mode_sock(vishost, visport);
for (int i=0; i<nev; i++)
{
if ( myid == 0 )
{
cout << "Eigenmode " << i+1 << '/' << nev
<< ", Lambda = " << eigenvalues[i] << endl;
}
// convert eigenvector from HypreParVector to ParGridFunction
x = lobpcg->GetEigenvector(i);
mode_sock << "parallel " << num_procs << " " << myid << "\n"
<< "solution\n" << *pmesh << x << flush
<< "window_title 'Eigenmode " << i+1 << '/' << nev
<< ", Lambda = " << eigenvalues[i] << "'" << endl;
char c;
if (myid == 0)
{
cout << "press (q)uit or (c)ontinue --> " << flush;
cin >> c;
}
MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
if (c != 'c')
{
break;
}
}
mode_sock.close();
}