SuperLURowLocMatrix::SuperLURowLocMatrix( const HypreParMatrix & hypParMat )
: comm_(hypParMat.GetComm()),
rowLocPtr_(NULL)
{
rowLocPtr_ = new SuperMatrix;
SuperMatrix * A = (SuperMatrix*)rowLocPtr_;
A->Store = NULL;
// First cast the parameter to a hypre_ParCSRMatrix
hypre_ParCSRMatrix * parcsr_op =
(hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(hypParMat);
MFEM_ASSERT(parcsr_op != NULL,"SuperLU: const_cast failed in SetOperator");
// Create the SuperMatrix A by borrowing the internal data from a
// hypre_CSRMatrix.
hypre_CSRMatrix * csr_op = hypre_MergeDiagAndOffd(parcsr_op);
hypre_CSRMatrixSetDataOwner(csr_op,0);
int m = parcsr_op->global_num_rows;
int n = parcsr_op->global_num_cols;
int fst_row = parcsr_op->first_row_index;
int nnz_loc = csr_op->num_nonzeros;
int m_loc = csr_op->num_rows;
height = m_loc;
width = m_loc;
double * nzval = csr_op->data;
int * colind = csr_op->j;
int * rowptr = NULL;
// The "i" array cannot be stolen from the hypre_CSRMatrix so we'll copy it
if ( !(rowptr = intMalloc_dist(m_loc+1)) )
{
ABORT("Malloc fails for rowptr[].");
}
for (int i=0; i<=m_loc; i++)
{
rowptr[i] = (csr_op->i)[i];
}
// Everything has been copied or abducted so delete the structure
hypre_CSRMatrixDestroy(csr_op);
// Assign he matrix data to SuperLU's SuperMatrix structure
dCreate_CompRowLoc_Matrix_dist(A, m, n, nnz_loc, m_loc, fst_row,
nzval, colind, rowptr,
SLU_NR_loc, SLU_D, SLU_GE);
}
int main(int argc, char *argv[])
{
// 1. Initialize MPI.
int num_procs, myid;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// 2. Parse command-line options.
const char *mesh_file = "../../data/star.mesh";
int order = 1;
bool set_bc = true;
bool static_cond = false;
bool hybridization = false;
bool visualization = 1;
bool use_petsc = true;
const char *petscrc_file = "";
bool use_nonoverlapping = false;
OptionsParser args(argc, argv);
args.AddOption(&mesh_file, "-m", "--mesh",
"Mesh file to use.");
args.AddOption(&order, "-o", "--order",
"Finite element order (polynomial degree).");
args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
"Impose or not essential boundary conditions.");
args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
" solution.");
args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
"--no-static-condensation", "Enable static condensation.");
args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
"--no-hybridization", "Enable hybridization.");
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
"Enable or disable GLVis visualization.");
args.AddOption(&use_petsc, "-usepetsc", "--usepetsc", "-no-petsc",
"--no-petsc",
"Use or not PETSc to solve the linear system.");
args.AddOption(&petscrc_file, "-petscopts", "--petscopts",
"PetscOptions file to use.");
args.AddOption(&use_nonoverlapping, "-nonoverlapping", "--nonoverlapping",
"-no-nonoverlapping", "--no-nonoverlapping",
"Use or not the block diagonal PETSc's matrix format "
"for non-overlapping domain decomposition.");
args.Parse();
if (!args.Good())
{
if (myid == 0)
{
args.PrintUsage(cout);
}
MPI_Finalize();
return 1;
}
if (myid == 0)
{
args.PrintOptions(cout);
}
// 2b. We initialize PETSc
if (use_petsc) { MFEMInitializePetsc(NULL,NULL,petscrc_file,NULL); }
kappa = freq * M_PI;
// 3. Read the (serial) mesh from the given mesh file on all processors. We
// can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
// and volume, as well as periodic meshes with the same code.
Mesh *mesh = new Mesh(mesh_file, 1, 1);
int dim = mesh->Dimension();
int sdim = mesh->SpaceDimension();
// 4. Refine the serial mesh on all processors to increase the resolution. In
// this example we do 'ref_levels' of uniform refinement. We choose
// 'ref_levels' to be the largest number that gives a final mesh with no
// more than 1,000 elements.
{
int ref_levels =
(int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
for (int l = 0; l < ref_levels; l++)
{
mesh->UniformRefinement();
}
}
// 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
// this mesh further in parallel to increase the resolution. Once the
// parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
// meshes need to be reoriented before we can define high-order Nedelec
// spaces on them (this is needed in the ADS solver below).
ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
delete mesh;
{
int par_ref_levels = 2;
for (int l = 0; l < par_ref_levels; l++)
{
pmesh->UniformRefinement();
}
}
pmesh->ReorientTetMesh();
// 6. Define a parallel finite element space on the parallel mesh. Here we
// use the Raviart-Thomas finite elements of the specified order.
FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
HYPRE_Int size = fespace->GlobalTrueVSize();
if (myid == 0)
{
cout << "Number of finite element unknowns: " << size << endl;
}
// 7. Determine the list of true (i.e. parallel conforming) essential
// boundary dofs. In this example, the boundary conditions are defined
// by marking all the boundary attributes from the mesh as essential
// (Dirichlet) and converting them to a list of true dofs.
Array<int> ess_tdof_list;
if (pmesh->bdr_attributes.Size())
{
Array<int> ess_bdr(pmesh->bdr_attributes.Max());
ess_bdr = set_bc ? 1 : 0;
fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
}
// 8. Set up the parallel linear form b(.) which corresponds to the
// right-hand side of the FEM linear system, which in this case is
// (f,phi_i) where f is given by the function f_exact and phi_i are the
// basis functions in the finite element fespace.
VectorFunctionCoefficient f(sdim, f_exact);
ParLinearForm *b = new ParLinearForm(fespace);
b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
b->Assemble();
// 9. Define the solution vector x as a parallel finite element grid function
// corresponding to fespace. Initialize x by projecting the exact
// solution. Note that only values from the boundary faces will be used
// when eliminating the non-homogeneous boundary condition to modify the
// r.h.s. vector b.
ParGridFunction x(fespace);
VectorFunctionCoefficient F(sdim, F_exact);
x.ProjectCoefficient(F);
// 10. Set up the parallel bilinear form corresponding to the H(div)
// diffusion operator grad alpha div + beta I, by adding the div-div and
// the mass domain integrators.
Coefficient *alpha = new ConstantCoefficient(1.0);
Coefficient *beta = new ConstantCoefficient(1.0);
ParBilinearForm *a = new ParBilinearForm(fespace);
a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
// 11. Assemble the parallel bilinear form and the corresponding linear
// system, applying any necessary transformations such as: parallel
// assembly, eliminating boundary conditions, applying conforming
// constraints for non-conforming AMR, static condensation,
// hybridization, etc.
FiniteElementCollection *hfec = NULL;
ParFiniteElementSpace *hfes = NULL;
if (static_cond)
{
a->EnableStaticCondensation();
}
else if (hybridization)
{
hfec = new DG_Interface_FECollection(order-1, dim);
hfes = new ParFiniteElementSpace(pmesh, hfec);
a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
ess_tdof_list);
}
a->Assemble();
Vector B, X;
CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
pcg->SetRelTol(1e-12);
pcg->SetMaxIter(500);
pcg->SetPrintLevel(1);
if (!use_petsc)
{
HypreParMatrix A;
a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
HYPRE_Int glob_size = A.GetGlobalNumRows();
if (myid == 0)
{
cout << "Size of linear system: " << glob_size << endl;
}
// 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
// the 3D ADS preconditioners from hypre. If using hybridization, the
// system is preconditioned with hypre's BoomerAMG.
HypreSolver *prec = NULL;
pcg->SetOperator(A);
if (hybridization) { prec = new HypreBoomerAMG(A); }
else
{
ParFiniteElementSpace *prec_fespace =
(a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
if (dim == 2) { prec = new HypreAMS(A, prec_fespace); }
else { prec = new HypreADS(A, prec_fespace); }
}
pcg->SetPreconditioner(*prec);
pcg->Mult(B, X);
delete prec;
}
else
{
PetscParMatrix A;
PetscPreconditioner *prec = NULL;
a->SetOperatorType(use_nonoverlapping ?
Operator::PETSC_MATIS : Operator::PETSC_MATAIJ);
a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
if (myid == 0)
{
cout << "Size of linear system: " << A.M() << endl;
}
pcg->SetOperator(A);
if (use_nonoverlapping)
{
ParFiniteElementSpace *prec_fespace =
(a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
// Auxiliary class for BDDC customization
PetscBDDCSolverParams opts;
// Inform the solver about the finite element space
opts.SetSpace(prec_fespace);
// Inform the solver about essential dofs
opts.SetEssBdrDofs(&ess_tdof_list);
// Create a BDDC solver with parameters
prec = new PetscBDDCSolver(A, opts);
}
else
{
// Create an empty preconditioner that can be customized at runtime.
prec = new PetscPreconditioner(A, "solver_");
}
pcg->SetPreconditioner(*prec);
pcg->Mult(B, X);
delete prec;
}
delete pcg;
// 13. Recover the parallel grid function corresponding to X. This is the
// local finite element solution on each processor.
a->RecoverFEMSolution(X, *b, x);
// 14. Compute and print the L^2 norm of the error.
{
double err = x.ComputeL2Error(F);
if (myid == 0)
{
cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
}
}
// 15. Save the refined mesh and the solution in parallel. This output can
// be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
{
ostringstream mesh_name, sol_name;
mesh_name << "mesh." << setfill('0') << setw(6) << myid;
sol_name << "sol." << setfill('0') << setw(6) << myid;
ofstream mesh_ofs(mesh_name.str().c_str());
mesh_ofs.precision(8);
pmesh->Print(mesh_ofs);
ofstream sol_ofs(sol_name.str().c_str());
sol_ofs.precision(8);
x.Save(sol_ofs);
}
// 16. Send the solution by socket to a GLVis server.
if (visualization)
{
char vishost[] = "localhost";
int visport = 19916;
socketstream sol_sock(vishost, visport);
sol_sock << "parallel " << num_procs << " " << myid << "\n";
sol_sock.precision(8);
sol_sock << "solution\n" << *pmesh << x << flush;
}
// 17. Free the used memory.
delete hfes;
delete hfec;
delete a;
delete alpha;
delete beta;
delete b;
delete fespace;
delete fec;
delete pmesh;
// We finalize PETSc
if (use_petsc) { MFEMFinalizePetsc(); }
MPI_Finalize();
return 0;
}
void
VoltaSolver::Solve()
{
if (myid_ == 0) { cout << "Running solver ... " << endl << flush; }
// Initialize the electric potential with its boundary conditions
*phi_ = 0.0;
if ( dbcs_->Size() > 0 )
{
if ( phiBCCoef_ )
{
// Apply gradient boundary condition
phi_->ProjectBdrCoefficient(*phiBCCoef_, ess_bdr_);
}
else
{
// Apply piecewise constant boundary condition
Array<int> dbc_bdr_attr(pmesh_->bdr_attributes.Max());
for (int i=0; i<dbcs_->Size(); i++)
{
ConstantCoefficient voltage((*dbcv_)[i]);
dbc_bdr_attr = 0;
dbc_bdr_attr[(*dbcs_)[i]-1] = 1;
phi_->ProjectBdrCoefficient(voltage, dbc_bdr_attr);
}
}
}
// Initialize the RHS vector
HypreParVector *RHS = new HypreParVector(H1FESpace_);
*RHS = 0.0;
// Initialize the volumetric charge density
if ( rho_ )
{
rho_->ProjectCoefficient(*rhoCoef_);
HypreParMatrix *MassH1 = h1Mass_->ParallelAssemble();
HypreParVector *Rho = rho_->ParallelProject();
MassH1->Mult(*Rho,*RHS);
delete MassH1;
delete Rho;
}
// Initialize the Polarization
HypreParVector *P = NULL;
if ( p_ )
{
p_->ProjectCoefficient(*pCoef_);
P = p_->ParallelProject();
HypreParMatrix *MassHCurl = hCurlMass_->ParallelAssemble();
HypreParVector *PD = new HypreParVector(HCurlFESpace_);
MassHCurl->Mult(*P,*PD);
Grad_->MultTranspose(*PD,*RHS,-1.0,1.0);
delete MassHCurl;
delete PD;
}
// Initialize the surface charge density
if ( sigma_ )
{
*sigma_ = 0.0;
Array<int> nbc_bdr_attr(pmesh_->bdr_attributes.Max());
for (int i=0; i<nbcs_->Size(); i++)
{
ConstantCoefficient sigma_coef((*nbcv_)[i]);
nbc_bdr_attr = 0;
nbc_bdr_attr[(*nbcs_)[i]-1] = 1;
sigma_->ProjectBdrCoefficient(sigma_coef, nbc_bdr_attr);
}
HypreParMatrix *MassS = h1SurfMass_->ParallelAssemble();
HypreParVector *Sigma = sigma_->ParallelProject();
MassS->Mult(*Sigma,*RHS,1.0,1.0);
delete MassS;
delete Sigma;
}
// Apply Dirichlet BCs to matrix and right hand side
HypreParMatrix *DivEpsGrad = divEpsGrad_->ParallelAssemble();
HypreParVector *Phi = phi_->ParallelProject();
// Apply the boundary conditions to the assembled matrix and vectors
if ( dbcs_->Size() > 0 )
{
// According to the selected surfaces
divEpsGrad_->ParallelEliminateEssentialBC(ess_bdr_,
*DivEpsGrad,
*Phi, *RHS);
}
else
{
// No surfaces were labeled as Dirichlet so eliminate one DoF
Array<int> dof_list(0);
if ( myid_ == 0 )
{
dof_list.SetSize(1);
dof_list[0] = 0;
}
DivEpsGrad->EliminateRowsCols(dof_list, *Phi, *RHS);
}
// Define and apply a parallel PCG solver for AX=B with the AMG
// preconditioner from hypre.
HypreSolver *amg = new HypreBoomerAMG(*DivEpsGrad);
HyprePCG *pcg = new HyprePCG(*DivEpsGrad);
pcg->SetTol(1e-12);
pcg->SetMaxIter(500);
pcg->SetPrintLevel(2);
pcg->SetPreconditioner(*amg);
pcg->Mult(*RHS, *Phi);
delete amg;
delete pcg;
delete DivEpsGrad;
delete RHS;
// Extract the parallel grid function corresponding to the finite
// element approximation Phi. This is the local solution on each
// processor.
*phi_ = *Phi;
// Compute the negative Gradient of the solution vector. This is
// the magnetic field corresponding to the scalar potential
// represented by phi.
HypreParVector *E = new HypreParVector(HCurlFESpace_);
Grad_->Mult(*Phi,*E,-1.0);
*e_ = *E;
delete Phi;
// Compute electric displacement (D) from E and P
if (myid_ == 0) { cout << "Computing D ... " << flush; }
HypreParMatrix *HCurlHDivEps = hCurlHDivEps_->ParallelAssemble();
HypreParVector *ED = new HypreParVector(HDivFESpace_);
HypreParVector *D = new HypreParVector(HDivFESpace_);
HCurlHDivEps->Mult(*E,*ED);
if ( P )
{
HypreParMatrix *HCurlHDiv = hCurlHDiv_->ParallelAssemble();
HCurlHDiv->Mult(*P,*ED,-1.0,1.0);
delete HCurlHDiv;
}
HypreParMatrix * MassHDiv = hDivMass_->ParallelAssemble();
HyprePCG * pcgM = new HyprePCG(*MassHDiv);
pcgM->SetTol(1e-12);
pcgM->SetMaxIter(500);
pcgM->SetPrintLevel(0);
HypreDiagScale *diagM = new HypreDiagScale;
pcgM->SetPreconditioner(*diagM);
pcgM->Mult(*ED,*D);
*d_ = *D;
if (myid_ == 0) { cout << "done." << flush; }
delete diagM;
delete pcgM;
delete HCurlHDivEps;
delete MassHDiv;
delete E;
delete ED;
delete D;
delete P;
if (myid_ == 0) { cout << " Solver done. " << flush; }
}
int main(int argc, char *argv[])
{
MPI_Session mpi(argc, argv);
// 1. Define and parse command-line options.
const char *mesh_file = "../data/beam-tri.mesh";
int ser_ref_levels = -1;
int par_ref_levels = 1;
int order = 1;
double alpha = -1.0;
double kappa = -1.0;
bool amg_elast = false;
bool visualization = 1;
OptionsParser args(argc, argv);
args.AddOption(&mesh_file, "-m", "--mesh",
"Mesh file to use.");
args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
"Number of times to refine the mesh uniformly before parallel"
" partitioning, -1 for auto.");
args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
"Number of times to refine the mesh uniformly after parallel"
" partitioning.");
args.AddOption(&order, "-o", "--order",
"Finite element order (polynomial degree).");
args.AddOption(&alpha, "-a", "--alpha",
"One of the two DG penalty parameters, typically +1/-1."
" See the documentation of class DGElasticityIntegrator.");
args.AddOption(&kappa, "-k", "--kappa",
"One of the two DG penalty parameters, should be positive."
" Negative values are replaced with (order+1)^2.");
args.AddOption(&amg_elast, "-elast", "--amg-for-elasticity", "-sys",
"--amg-for-systems",
"Use the special AMG elasticity solver (GM/LN approaches), "
"or standard AMG for systems (unknown approach).");
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
"Enable or disable GLVis visualization.");
args.Parse();
if (!args.Good())
{
if (mpi.Root()) { args.PrintUsage(cout); }
return 1;
}
if (kappa < 0)
{
kappa = (order+1)*(order+1);
}
if (mpi.Root()) { args.PrintOptions(cout); }
// 2. Read the mesh from the given mesh file.
Mesh mesh(mesh_file, 1, 1);
int dim = mesh.Dimension();
if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
{
if (mpi.Root())
{
cerr << "\nInput mesh should have at least two materials and "
<< "two boundary attributes! (See schematic in ex17p.cpp)\n"
<< endl;
}
return 3;
}
// 3. Refine the mesh to increase the resolution.
if (ser_ref_levels < 0)
{
ser_ref_levels = (int)floor(log(5000./mesh.GetNE())/log(2.)/dim);
}
for (int l = 0; l < ser_ref_levels; l++)
{
mesh.UniformRefinement();
}
// Since NURBS meshes do not support DG integrators, we convert them to
// regular polynomial mesh of the specified (solution) order.
if (mesh.NURBSext) { mesh.SetCurvature(order); }
ParMesh pmesh(MPI_COMM_WORLD, mesh);
mesh.Clear();
for (int l = 0; l < par_ref_levels; l++)
{
pmesh.UniformRefinement();
}
// 4. Define a DG vector finite element space on the mesh. Here, we use
// Gauss-Lobatto nodal basis because it gives rise to a sparser matrix
// compared to the default Gauss-Legendre nodal basis.
DG_FECollection fec(order, dim, BasisType::GaussLobatto);
ParFiniteElementSpace fespace(&pmesh, &fec, dim, Ordering::byVDIM);
HYPRE_Int glob_size = fespace.GlobalTrueVSize();
if (mpi.Root())
{
cout << "Number of finite element unknowns: " << glob_size
<< "\nAssembling: " << flush;
}
// 5. In this example, the Dirichlet boundary conditions are defined by
// marking boundary attributes 1 and 2 in the marker Array 'dir_bdr'.
// These b.c. are imposed weakly, by adding the appropriate boundary
// integrators over the marked 'dir_bdr' to the bilinear and linear forms.
// With this DG formulation, there are no essential boundary conditions.
Array<int> ess_tdof_list; // no essential b.c. (empty list)
Array<int> dir_bdr(pmesh.bdr_attributes.Max());
dir_bdr = 0;
dir_bdr[0] = 1; // boundary attribute 1 is Dirichlet
dir_bdr[1] = 1; // boundary attribute 2 is Dirichlet
// 6. Define the DG solution vector 'x' as a finite element grid function
// corresponding to fespace. Initialize 'x' using the 'InitDisplacement'
// function.
ParGridFunction x(&fespace);
VectorFunctionCoefficient init_x(dim, InitDisplacement);
x.ProjectCoefficient(init_x);
// 7. Set up the Lame constants for the two materials. They are defined as
// piece-wise (with respect to the element attributes) constant
// coefficients, i.e. type PWConstCoefficient.
Vector lambda(pmesh.attributes.Max());
lambda = 1.0; // Set lambda = 1 for all element attributes.
lambda(0) = 50.0; // Set lambda = 50 for element attribute 1.
PWConstCoefficient lambda_c(lambda);
Vector mu(pmesh.attributes.Max());
mu = 1.0; // Set mu = 1 for all element attributes.
mu(0) = 50.0; // Set mu = 50 for element attribute 1.
PWConstCoefficient mu_c(mu);
// 8. Set up the linear form b(.) which corresponds to the right-hand side of
// the FEM linear system. In this example, the linear form b(.) consists
// only of the terms responsible for imposing weakly the Dirichlet
// boundary conditions, over the attributes marked in 'dir_bdr'. The
// values for the Dirichlet boundary condition are taken from the
// VectorFunctionCoefficient 'x_init' which in turn is based on the
// function 'InitDisplacement'.
ParLinearForm b(&fespace);
if (mpi.Root()) { cout << "r.h.s. ... " << flush; }
b.AddBdrFaceIntegrator(
new DGElasticityDirichletLFIntegrator(
init_x, lambda_c, mu_c, alpha, kappa), dir_bdr);
b.Assemble();
// 9. Set up the bilinear form a(.,.) on the DG finite element space
// corresponding to the linear elasticity integrator with coefficients
// lambda and mu as defined above. The additional interior face integrator
// ensures the weak continuity of the displacement field. The additional
// boundary face integrator works together with the boundary integrator
// added to the linear form b(.) to impose weakly the Dirichlet boundary
// conditions.
ParBilinearForm a(&fespace);
a.AddDomainIntegrator(new ElasticityIntegrator(lambda_c, mu_c));
a.AddInteriorFaceIntegrator(
new DGElasticityIntegrator(lambda_c, mu_c, alpha, kappa));
a.AddBdrFaceIntegrator(
new DGElasticityIntegrator(lambda_c, mu_c, alpha, kappa), dir_bdr);
// 10. Assemble the bilinear form and the corresponding linear system.
if (mpi.Root()) { cout << "matrix ... " << flush; }
a.Assemble();
HypreParMatrix A;
Vector B, X;
a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
if (mpi.Root()) { cout << "done." << endl; }
// 11. Define a simple symmetric Gauss-Seidel preconditioner and use it to
// solve the system Ax=b with PCG for the symmetric formulation, or GMRES
// for the non-symmetric.
const double rtol = 1e-6;
HypreBoomerAMG amg(A);
if (amg_elast)
{
amg.SetElasticityOptions(&fespace);
}
else
{
amg.SetSystemsOptions(dim);
}
CGSolver pcg(A.GetComm());
GMRESSolver gmres(A.GetComm());
gmres.SetKDim(50);
IterativeSolver &ipcg = pcg, &igmres = gmres;
IterativeSolver &solver = (alpha == -1.0) ? ipcg : igmres;
solver.SetRelTol(rtol);
solver.SetMaxIter(500);
solver.SetPrintLevel(1);
solver.SetOperator(A);
solver.SetPreconditioner(amg);
solver.Mult(B, X);
// 12. Recover the solution as a finite element grid function 'x'.
a.RecoverFEMSolution(X, b, x);
// 13. Use the DG solution space as the mesh nodal space. This allows us to
// save the displaced mesh as a curved DG mesh.
pmesh.SetNodalFESpace(&fespace);
Vector reference_nodes;
if (visualization) { reference_nodes = *pmesh.GetNodes(); }
// 14. Save the displaced mesh and minus the solution (which gives the
// backward displacements to the reference mesh). This output can be
// viewed later using GLVis: "glvis -m displaced.mesh -g sol.gf".
{
*pmesh.GetNodes() += x;
x.Neg(); // x = -x
ostringstream mesh_name, sol_name;
mesh_name << "mesh." << setfill('0') << setw(6) << mpi.WorldRank();
sol_name << "sol." << setfill('0') << setw(6) << mpi.WorldRank();
ofstream mesh_ofs(mesh_name.str().c_str());
mesh_ofs.precision(8);
mesh_ofs << pmesh;
ofstream sol_ofs(sol_name.str().c_str());
sol_ofs.precision(8);
sol_ofs << x;
}
// 15. Visualization: send data by socket to a GLVis server.
if (visualization)
{
char vishost[] = "localhost";
int visport = 19916;
VisMan vis(vishost, visport);
const char *glvis_keys = (dim < 3) ? "Rjlc" : "c";
// Visualize the deformed configuration.
vis << new_window << setprecision(8)
<< "parallel " << pmesh.GetNRanks() << ' ' << pmesh.GetMyRank()
<< '\n'
<< "solution\n" << pmesh << x << flush
<< "keys " << glvis_keys << endl
<< "window_title 'Deformed configuration'" << endl
<< "plot_caption 'Backward displacement'" << endl
<< position_window << close_connection;
// Visualize the stress components.
const char *c = "xyz";
ParFiniteElementSpace scalar_dg_space(&pmesh, &fec);
ParGridFunction stress(&scalar_dg_space);
StressCoefficient stress_c(lambda_c, mu_c);
*pmesh.GetNodes() = reference_nodes;
x.Neg(); // x = -x
stress_c.SetDisplacement(x);
for (int si = 0; si < dim; si++)
{
for (int sj = si; sj < dim; sj++)
{
stress_c.SetComponent(si, sj);
stress.ProjectCoefficient(stress_c);
MPI_Barrier(MPI_COMM_WORLD);
vis << new_window << setprecision(8)
<< "parallel " << pmesh.GetNRanks() << ' ' << pmesh.GetMyRank()
<< '\n'
<< "solution\n" << pmesh << stress << flush
<< "keys " << glvis_keys << endl
<< "window_title |Stress " << c[si] << c[sj] << '|' << endl
<< position_window << close_connection;
}
}
}
return 0;
}
void
TeslaSolver::Solve()
{
if (myid_ == 0) { cout << "Running solver ... " << endl << flush; }
// Initialize the magnetic vector potential with its boundary conditions
*a_ = 0.0;
// Apply surface currents if available
if ( k_ )
{
SurfCur_->ComputeSurfaceCurrent(*k_);
*a_ = *k_;
}
// Apply uniform B boundary condition on remaining surfaces
a_->ProjectBdrCoefficientTangent(*aBCCoef_, non_k_bdr_);
// Initialize the RHS vector
HypreParVector *RHS = new HypreParVector(HCurlFESpace_);
*RHS = 0.0;
HypreParMatrix *MassHCurl = hCurlMass_->ParallelAssemble();
// Initialize the volumetric current density
if ( j_ )
{
j_->ProjectCoefficient(*jCoef_);
HypreParVector *J = j_->ParallelProject();
HypreParVector *JD = new HypreParVector(HCurlFESpace_);
MassHCurl->Mult(*J,*JD);
DivFreeProj_->Mult(*JD, *RHS);
delete J;
delete JD;
}
// Initialize the Magnetization
HypreParVector *M = NULL;
if ( m_ )
{
m_->ProjectCoefficient(*mCoef_);
M = m_->ParallelProject();
HypreParMatrix *MassHDiv = hDivMassMuInv_->ParallelAssemble();
HypreParVector *MD = new HypreParVector(HDivFESpace_);
MassHDiv->Mult(*M,*MD);
Curl_->MultTranspose(*MD,*RHS,mu0_,1.0);
delete MassHDiv;
delete MD;
}
// Apply Dirichlet BCs to matrix and right hand side
HypreParMatrix *CurlMuInvCurl = curlMuInvCurl_->ParallelAssemble();
HypreParVector *A = a_->ParallelProject();
// Apply the boundary conditions to the assembled matrix and vectors
curlMuInvCurl_->ParallelEliminateEssentialBC(ess_bdr_,
*CurlMuInvCurl,
*A, *RHS);
// Define and apply a parallel PCG solver for AX=B with the AMS
// preconditioner from hypre.
HypreAMS *ams = new HypreAMS(*CurlMuInvCurl, HCurlFESpace_);
ams->SetSingularProblem();
HyprePCG *pcg = new HyprePCG(*CurlMuInvCurl);
pcg->SetTol(1e-12);
pcg->SetMaxIter(500);
pcg->SetPrintLevel(2);
pcg->SetPreconditioner(*ams);
pcg->Mult(*RHS, *A);
delete ams;
delete pcg;
delete CurlMuInvCurl;
delete RHS;
// Extract the parallel grid function corresponding to the finite
// element approximation Phi. This is the local solution on each
// processor.
*a_ = *A;
// Compute the negative Gradient of the solution vector. This is
// the magnetic field corresponding to the scalar potential
// represented by phi.
HypreParVector *B = new HypreParVector(HDivFESpace_);
Curl_->Mult(*A,*B);
*b_ = *B;
// Compute magnetic field (H) from B and M
if (myid_ == 0) { cout << "Computing H ... " << flush; }
HypreParMatrix *HDivHCurlMuInv = hDivHCurlMuInv_->ParallelAssemble();
HypreParVector *BD = new HypreParVector(HCurlFESpace_);
HypreParVector *H = new HypreParVector(HCurlFESpace_);
HDivHCurlMuInv->Mult(*B,*BD);
if ( M )
{
HDivHCurlMuInv->Mult(*M,*BD,-1.0*mu0_,1.0);
}
HyprePCG * pcgM = new HyprePCG(*MassHCurl);
pcgM->SetTol(1e-12);
pcgM->SetMaxIter(500);
pcgM->SetPrintLevel(0);
HypreDiagScale *diagM = new HypreDiagScale;
pcgM->SetPreconditioner(*diagM);
pcgM->Mult(*BD,*H);
*h_ = *H;
if (myid_ == 0) { cout << "done." << flush; }
delete diagM;
delete pcgM;
delete HDivHCurlMuInv;
delete MassHCurl;
delete A;
delete B;
delete BD;
delete H;
delete M;
if (myid_ == 0) { cout << " Solver done. " << flush; }
}
int main(int argc, char *argv[])
{
// 0. Initialize MPI.
int num_procs, myid;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// 1. Parse command-line options.
const char *mesh_file = "../data/beam-tri.mesh";
int serial_ref_levels = 0;
int order = 1;
bool static_cond = false;
bool visualization = 1;
OptionsParser args(argc, argv);
args.AddOption(&mesh_file, "-m", "--mesh",
"Mesh file to use.");
args.AddOption(&serial_ref_levels, "-rs", "--refine-serial",
"Number of uniform serial refinements (before parallel"
" partitioning)");
args.AddOption(&order, "-o", "--order",
"Finite element order (polynomial degree).");
args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
"--no-static-condensation", "Enable static condensation.");
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
"Enable or disable GLVis visualization.");
args.Parse();
if (!args.Good())
{
if (myid == 0)
{
args.PrintUsage(cout);
}
MPI_Finalize();
return 1;
}
if (myid == 0)
{
args.PrintOptions(cout);
}
// 2. Read the mesh from the given mesh file. We can handle triangular,
// quadrilateral, tetrahedral, and hexahedral meshes with the same code.
Mesh mesh(mesh_file, 1, 1);
int dim = mesh.Dimension();
MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");
if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
{
cerr << "\nInput mesh should have at least two materials and "
<< "two boundary attributes! (See schematic in ex2.cpp)\n"
<< endl;
MPI_Finalize();
return 3;
}
// 3. Refine the mesh before parallel partitioning. Since a NURBS mesh can
// currently only be refined uniformly, we need to convert it to a
// piecewise-polynomial curved mesh. First we refine the NURBS mesh a bit
// more and then project the curvature to quadratic Nodes.
if (mesh.NURBSext && serial_ref_levels == 0)
{
serial_ref_levels = 2;
}
for (int i = 0; i < serial_ref_levels; i++)
{
mesh.UniformRefinement();
}
if (mesh.NURBSext)
{
mesh.SetCurvature(2);
}
mesh.EnsureNCMesh();
ParMesh pmesh(MPI_COMM_WORLD, mesh);
mesh.Clear();
// 4. Define a finite element space on the mesh. The polynomial order is
// one (linear) by default, but this can be changed on the command line.
H1_FECollection fec(order, dim);
ParFiniteElementSpace fespace(&pmesh, &fec, dim);
// 5. As in Example 2, we set up the linear form b(.) which corresponds to
// the right-hand side of the FEM linear system. In this case, b_i equals
// the boundary integral of f*phi_i where f represents a "pull down"
// force on the Neumann part of the boundary and phi_i are the basis
// functions in the finite element fespace. The force is defined by the
// VectorArrayCoefficient object f, which is a vector of Coefficient
// objects. The fact that f is non-zero on boundary attribute 2 is
// indicated by the use of piece-wise constants coefficient for its last
// component. We don't assemble the discrete problem yet, this will be
// done in the main loop.
VectorArrayCoefficient f(dim);
for (int i = 0; i < dim-1; i++)
{
f.Set(i, new ConstantCoefficient(0.0));
}
{
Vector pull_force(pmesh.bdr_attributes.Max());
pull_force = 0.0;
pull_force(1) = -1.0e-2;
f.Set(dim-1, new PWConstCoefficient(pull_force));
}
ParLinearForm b(&fespace);
b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
// 6. Set up the bilinear form a(.,.) on the finite element space
// corresponding to the linear elasticity integrator with piece-wise
// constants coefficient lambda and mu.
Vector lambda(pmesh.attributes.Max());
lambda = 1.0;
lambda(0) = lambda(1)*50;
PWConstCoefficient lambda_func(lambda);
Vector mu(pmesh.attributes.Max());
mu = 1.0;
mu(0) = mu(1)*50;
PWConstCoefficient mu_func(mu);
ParBilinearForm a(&fespace);
BilinearFormIntegrator *integ =
new ElasticityIntegrator(lambda_func,mu_func);
a.AddDomainIntegrator(integ);
if (static_cond) { a.EnableStaticCondensation(); }
// 7. The solution vector x and the associated finite element grid function
// will be maintained over the AMR iterations. We initialize it to zero.
Vector zero_vec(dim);
zero_vec = 0.0;
VectorConstantCoefficient zero_vec_coeff(zero_vec);
ParGridFunction x(&fespace);
x = 0.0;
// 8. Determine the list of true (i.e. conforming) essential boundary dofs.
// In this example, the boundary conditions are defined by marking only
// boundary attribute 1 from the mesh as essential and converting it to a
// list of true dofs. The conversion to true dofs will be done in the
// main loop.
Array<int> ess_bdr(pmesh.bdr_attributes.Max());
ess_bdr = 0;
ess_bdr[0] = 1;
// 9. GLVis visualization.
char vishost[] = "localhost";
int visport = 19916;
socketstream sol_sock;
// 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
// that uses the ComputeElementFlux method of the ElasticityIntegrator to
// recover a smoothed flux (stress) that is subtracted from the element
// flux to get an error indicator. We need to supply the space for the
// smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
// Here, tdim represents the number of components for a symmetric (dim x
// dim) tensor.
const int tdim = dim*(dim+1)/2;
L2_FECollection flux_fec(order, dim);
ParFiniteElementSpace flux_fespace(&pmesh, &flux_fec, tdim);
ParFiniteElementSpace smooth_flux_fespace(&pmesh, &fec, tdim);
L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace,
smooth_flux_fespace);
// 11. A refiner selects and refines elements based on a refinement strategy.
// The strategy here is to refine elements with errors larger than a
// fraction of the maximum element error. Other strategies are possible.
// The refiner will call the given error estimator.
ThresholdRefiner refiner(estimator);
refiner.SetTotalErrorFraction(0.7);
// 12. The main AMR loop. In each iteration we solve the problem on the
// current mesh, visualize the solution, and refine the mesh.
const int max_dofs = 50000;
const int max_amr_itr = 20;
for (int it = 0; it <= max_amr_itr; it++)
{
HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
if (myid == 0)
{
cout << "\nAMR iteration " << it << endl;
cout << "Number of unknowns: " << global_dofs << endl;
}
// 13. Assemble the stiffness matrix and the right-hand side.
a.Assemble();
b.Assemble();
// 14. Set Dirichlet boundary values in the GridFunction x.
// Determine the list of Dirichlet true DOFs in the linear system.
Array<int> ess_tdof_list;
x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
// 15. Create the linear system: eliminate boundary conditions, constrain
// hanging nodes and possibly apply other transformations. The system
// will be solved for true (unconstrained) DOFs only.
HypreParMatrix A;
Vector B, X;
const int copy_interior = 1;
a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
// 16. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
// preconditioner from hypre.
HypreBoomerAMG amg;
amg.SetPrintLevel(0);
// amg.SetSystemsOptions(dim); // optional
CGSolver pcg(A.GetComm());
pcg.SetPreconditioner(amg);
pcg.SetOperator(A);
pcg.SetRelTol(1e-6);
pcg.SetMaxIter(500);
pcg.SetPrintLevel(3); // print the first and the last iterations only
pcg.Mult(B, X);
// 17. After solving the linear system, reconstruct the solution as a
// finite element GridFunction. Constrained nodes are interpolated
// from true DOFs (it may therefore happen that x.Size() >= X.Size()).
a.RecoverFEMSolution(X, b, x);
// 18. Send solution by socket to the GLVis server.
if (visualization && it == 0)
{
sol_sock.open(vishost, visport);
sol_sock.precision(8);
}
if (visualization && sol_sock.good())
{
GridFunction nodes(&fespace), *nodes_p = &nodes;
pmesh.GetNodes(nodes);
nodes += x;
int own_nodes = 0;
pmesh.SwapNodes(nodes_p, own_nodes);
x.Neg(); // visualize the backward displacement
sol_sock << "parallel " << num_procs << ' ' << myid << '\n';
sol_sock << "solution\n" << pmesh << x << flush;
x.Neg();
pmesh.SwapNodes(nodes_p, own_nodes);
if (it == 0)
{
sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
}
sol_sock << "window_title 'AMR iteration: " << it << "'\n"
<< "pause" << endl;
if (myid == 0)
{
cout << "Visualization paused. "
"Press <space> in the GLVis window to continue." << endl;
}
}
if (global_dofs > max_dofs)
{
if (myid == 0)
{
cout << "Reached the maximum number of dofs. Stop." << endl;
}
break;
}
// 19. Call the refiner to modify the mesh. The refiner calls the error
// estimator to obtain element errors, then it selects elements to be
// refined and finally it modifies the mesh. The Stop() method can be
// used to determine if a stopping criterion was met.
refiner.Apply(pmesh);
if (refiner.Stop())
{
if (myid == 0)
{
cout << "Stopping criterion satisfied. Stop." << endl;
}
break;
}
// 20. Update the space to reflect the new state of the mesh. Also,
// interpolate the solution x so that it lies in the new space but
// represents the same function. This saves solver iterations later
// since we'll have a good initial guess of x in the next step.
// Internally, FiniteElementSpace::Update() calculates an
// interpolation matrix which is then used by GridFunction::Update().
fespace.Update();
x.Update();
// 21. Load balance the mesh, and update the space and solution. Currently
// available only for nonconforming meshes.
if (pmesh.Nonconforming())
{
pmesh.Rebalance();
// Update the space and the GridFunction. This time the update matrix
// redistributes the GridFunction among the processors.
fespace.Update();
x.Update();
}
// 22. Inform also the bilinear and linear forms that the space has
// changed.
a.Update();
b.Update();
}
{
ostringstream mref_name, mesh_name, sol_name;
mref_name << "ex22p_reference_mesh." << setfill('0') << setw(6) << myid;
mesh_name << "ex22p_deformed_mesh." << setfill('0') << setw(6) << myid;
sol_name << "ex22p_displacement." << setfill('0') << setw(6) << myid;
ofstream mesh_ref_out(mref_name.str().c_str());
mesh_ref_out.precision(16);
pmesh.Print(mesh_ref_out);
ofstream mesh_out(mesh_name.str().c_str());
mesh_out.precision(16);
GridFunction nodes(&fespace), *nodes_p = &nodes;
pmesh.GetNodes(nodes);
nodes += x;
int own_nodes = 0;
pmesh.SwapNodes(nodes_p, own_nodes);
pmesh.Print(mesh_out);
pmesh.SwapNodes(nodes_p, own_nodes);
ofstream x_out(sol_name.str().c_str());
x_out.precision(16);
x.Save(x_out);
}
MPI_Finalize();
return 0;
}
