// ================================================ ====== ==== ==== == =
// Forms the coarse matrix, given the prolongator
int ML_Epetra::FaceMatrixFreePreconditioner::FormCoarseMatrix()
{
CoarseMat_ML = ML_Operator_Create(ml_comm_);
CoarseMat_ML->data_destroy=free;
ML_Operator *Temp_ML=0;
ML_Operator *R= ML_Operator_Create(ml_comm_);
ML_Operator *P= ML_Operator_Create(ml_comm_);
/* Build ML_Operator version of Prolongator_, Restriction Operator */
ML_CHK_ERR(ML_Operator_WrapEpetraCrsMatrix(Prolongator_,P,verbose_));
P->num_rigid=P->num_PDEs=dim;
//NTS: ML_CHK_ERR won't work on this: it returns 1
ML_Operator_Transpose_byrow(P, R);
/* OPTION: Disable the addon */
const Epetra_CrsMatrix *Op11crs = dynamic_cast<const Epetra_CrsMatrix*>(&*Operator_);
const Epetra_Operator_With_MatMat *Op11mm = dynamic_cast<const Epetra_Operator_With_MatMat*>(&*Operator_);
/* Do the A*P with or without addon*/
if(Op11crs){
if(verbose_ && !Comm_->MyPID()) printf("FMFP: Running *without* addon\n");
ML_Operator *SM_ML = ML_Operator_Create(ml_comm_);
Temp_ML = ML_Operator_Create(ml_comm_);
ML_Operator_WrapEpetraCrsMatrix((Epetra_CrsMatrix*)Op11crs,SM_ML,verbose_);
ML_2matmult(SM_ML,P,Temp_ML,ML_CSR_MATRIX);
ML_Operator_Destroy(&SM_ML);
}
else if(Op11mm){
if(verbose_ && !Comm_->MyPID()) printf("FMFP: Running with addon\n");
ML_CHK_ERR(Op11mm->MatrixMatrix_Multiply(*Prolongator_,ml_comm_,&Temp_ML));
}
else{
if(!Comm_->MyPID()) printf("ERROR: FMFP Illegal Operator\n");
delete R;
ML_CHK_ERR(-1);
}
/* Do R * AP */
R->num_rigid=R->num_PDEs=dim;
ML_2matmult_block(R, Temp_ML,CoarseMat_ML,ML_CSR_MATRIX);
/* Wrap to Epetra-land */
int nnz=100;
double time;
ML_Operator2EpetraCrsMatrix(CoarseMat_ML,CoarseMatrix,nnz,true,time,0,verbose_);
// NTS: This is a hack to get around the sticking ones on the diagonal issue;
/* Cleanup */
ML_Operator_Destroy(&P);
ML_Operator_Destroy(&R);
ML_Operator_Destroy(&Temp_ML);
ML_Operator_Destroy(&CoarseMat_ML);CoarseMat_ML=0;//HAX
return 0;
}/*end FormCoarseMatrix*/
/*----------------------------------------------------------------------*
| Constructor (public) m.gee 12/04|
| IMPORTANT: |
| No matter on which level we are here, the vector xfine is ALWAYS |
| a fine grid vector here! |
*----------------------------------------------------------------------*/
ML_NOX::ML_Nox_MatrixfreeLevel::ML_Nox_MatrixfreeLevel(int level, int nlevel, int plevel,
ML* ml, ML_Aggregate* ag, Epetra_CrsMatrix** P,
ML_NOX::Ml_Nox_Fineinterface& interface, const Epetra_Comm& comm,
const Epetra_Vector& xfine, double fd_alpha, double fd_beta,
bool fd_centered, bool isDiagonalOnly, int bsize)
: fineinterface_(interface),
comm_(comm)
{
level_ = level;
nlevel_ = nlevel;
ml_printlevel_ = plevel;
ml_ = ml;
ag_ = ag;
fd_alpha_ = fd_alpha;
fd_beta_ = fd_beta;
fd_centered_ = fd_centered;
isDiagonalOnly_ = isDiagonalOnly;
A_ = 0;
coarseinterface_= 0;
bsize_ = bsize;
// we need the graph of the operator on this level. On the fine grid we can just ask the
// fineinterface for it, on the coarser levels it has to be extracted from the ML-hierachy
if (level_==0)
{
// the Epetra_CrsGraph-copy-constructor shares data with the original one.
// We want a really deep copy here so we cannot use it
// graph_ will be given to the FiniteDifferencing class and will be destroyed by it
graph_ = ML_NOX::deepcopy_graph(interface.getGraph());
}
else
{
// Note that ML has no understanding of global indices, so it makes up new GIDs
// (This also holds for the Prolongators P)
Epetra_CrsMatrix* tmpMat = 0;
int maxnnz = 0;
double cputime = 0.0;
ML_Operator2EpetraCrsMatrix(&(ml_->Amat[level_]), tmpMat, maxnnz, false, cputime);
// copy the graph
double t0 = GetClock();
graph_ = ML_NOX::deepcopy_graph(&(tmpMat->Graph()));
// delete the copy of the Epetra_CrsMatrix
if (tmpMat) delete tmpMat; tmpMat = 0;
double t1 = GetClock();
if (ml_printlevel_ > 0 && 0 == comm_.MyPID())
cout << "matrixfreeML (level " << level_ << "): extraction/copy of Graph in " << cputime+t1-t0 << " sec\n"
<< " max-nonzeros in Graph: " << maxnnz << "\n";
}
// create this levels coarse interface
coarseinterface_ = new ML_NOX::Nox_CoarseProblem_Interface(fineinterface_,level_,ml_printlevel_,
P,&(graph_->RowMap()),nlevel_);
// restrict the xfine-vector to this level
Epetra_Vector* xthis = coarseinterface_->restrict_fine_to_this(xfine);
if (!xthis)
{
cout << "**ERR**: ML_Epetra::ML_Nox_MatrixfreeLevel::ML_Nox_MatrixfreeLevel:\n"
<< "**ERR**: ML_Epetra::Nox_CoarseProblem_Interface::restrict_fine_to_this returned NULL on level "
<< level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
Epetra_Vector* xc = new Epetra_Vector(graph_->RowMap(),false);
// FIXME: after intesive testing, this test might be obsolet
#if 0
bool samemap = xc->Map().PointSameAs(xthis->Map());
if (samemap)
{
#endif
xc->Update(1.0,*xthis,0.0);
#if 0
}
else
{
cout << "**WRN** Maps are not equal in\n"
<< "**WRN** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
// import the xthis vector in the Map that ML produced for graph_
Epetra_Import* importer = new Epetra_Import(graph_->RowMap(),xthis->Map());
int ierr = xc->Import(*xthis,*importer,Insert);
if (ierr)
{
cout << "**ERR**: ML_Epetra::ML_Nox_MatrixfreeLevel::ML_Nox_MatrixfreeLevel:\n"
<< "**ERR**: export from xthis to xc returned err=" << ierr <<"\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
if (importer) delete importer; importer = 0;
}
#endif
if (xthis) delete xthis; xthis = 0;
// create the coloring of the graph
if (ml_printlevel_>0 && comm_.MyPID()==0)
{
cout << "matrixfreeML (level " << level_ << "): Entering Coloring on level " << level_ << "\n";
fflush(stdout);
}
double t0 = GetClock();
colorMap_ = ML_NOX::ML_Nox_collapsedcoloring(graph_,bsize_,isDiagonalOnly,ml_printlevel_);
if (!colorMap_) colorMap_ = ML_NOX::ML_Nox_standardcoloring(graph_,isDiagonalOnly);
colorMapIndex_ = new EpetraExt::CrsGraph_MapColoringIndex(*colorMap_);
colorcolumns_ = &(*colorMapIndex_)(*graph_);
double t1 = GetClock();
if (ml_printlevel_>0 && comm_.MyPID()==0)
{
cout << "matrixfreeML (level " << level_ << "): Proc " << comm_.MyPID() <<" Coloring time is " << (t1-t0) << " sec\n";
fflush(stdout);
}
// construct the FiniteDifferenceColoring-Matrix
if (ml_printlevel_>0 && comm_.MyPID()==0)
{
cout << "matrixfreeML (level " << level_ << "): Entering Construction FD-Operator on level " << level_ << "\n";
fflush(stdout);
}
t0 = GetClock();
#if 1 // FD-operator with coloring
FD_ = new NOX::EpetraNew::FiniteDifferenceColoring(*coarseinterface_,
*xc,
*graph_,
*colorMap_,
*colorcolumns_,
true,
isDiagonalOnly_,
fd_beta_,fd_alpha_);
#else // FD-operator without coloring
FD_ = new NOX::EpetraNew::FiniteDifference(*coarseinterface_,
*xc,
*graph_,
fd_beta_,fd_alpha_);
#endif
// set differencing method
if (fd_centered_)
FD_->setDifferenceMethod(NOX::EpetraNew::FiniteDifferenceColoring::Centered);
bool err = FD_->computeJacobian(*xc);
if (err==false)
{
cout << "**ERR**: ML_NOX::ML_Nox_MatrixfreeLevel::ML_Nox_MatrixfreeLevel:\n"
<< "**ERR**: NOX::Epetra::FiniteDifferenceColoring returned an error on level "
<< level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// print number of calls to the coarse interface
if (ml_printlevel_>0 && comm_.MyPID()==0)
cout << "matrixfreeML (level " << level_ << "): Calls to coarse-computeF in FD-Operator: "
<< coarseinterface_->numcallscomputeF() << "\n";
t1 = GetClock();
if (ml_printlevel_>0 && comm_.MyPID()==0)
{
cout << "matrixfreeML (level " << level_ << "): Proc " << comm_.MyPID() <<" colored Finite Differencing time is " << (t1-t0) << " sec\n";
fflush(stdout);
}
// get ref to computed Epetra_CrsMatrix
A_ = dynamic_cast<Epetra_CrsMatrix*>(&(FD_->getUnderlyingMatrix()));
// set counter for number of calls to the coarseinterface_->computeF back to zero
coarseinterface_->resetnumcallscomputeF();
// tidy up
if (xc) delete xc; xc = 0;
return;
}
/*----------------------------------------------------------------------*
| recreate this level (public) m.gee 01/05|
| this function assumes, that the graph of the fine level problem has |
| not changed since call to the constructor and therefore |
| the graph and it's coloring do not have to be recomputed |
| IMPORTANT: |
| No matter on which level we are here, the vector xfine is ALWAYS |
| a fine grid vector here! |
*----------------------------------------------------------------------*/
bool ML_NOX::ML_Nox_MatrixfreeLevel::recreateLevel(int level, int nlevel, int plevel,
ML* ml, ML_Aggregate* ag, Epetra_CrsMatrix** P,
ML_NOX::Ml_Nox_Fineinterface& interface,
const Epetra_Comm& comm, const Epetra_Vector& xfine)
{
// make some tests
if (level != level_)
{
cout << "**ERR**: ML_Epetra::ML_Nox_MatrixfreeLevel::recreateLevel:\n"
<< "**ERR**: level_ " << level_ << " not equal level " << level << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
if (nlevel != nlevel_)
{
cout << "**ERR**: ML_Epetra::ML_Nox_MatrixfreeLevel::recreateLevel:\n"
<< "**ERR**: nlevel_ " << nlevel_ << " not equal nlevel " << nlevel << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// printlevel might have changed
ml_printlevel_ = plevel;
ml_ = ml;
ag_ = ag;
destroyP(); // safer to use the new Ps
setP(NULL);
// we need the graph of the operator on this level. On the fine grid we can just ask the
// fineinterface for it, on the coarser levels it has to be extracted from the ML-hierachy
bool same;
if (level_==0)
{
const Epetra_CrsGraph* graph = interface.getGraph();
// check whether the old graph matches the new one
same = compare_graphs(graph,graph_);
destroyFD(); // we are here to recompute the FD-operator (this destroys graph_)
graph_ = ML_NOX::deepcopy_graph(graph);
}
else
{
// Note that ML has no understanding of global indices, so it makes up new GIDs
// (This also holds for the Prolongators P)
Epetra_CrsMatrix* tmpMat = 0;
int maxnnz = 0;
double cputime = 0.0;
ML_Operator2EpetraCrsMatrix(&(ml_->Amat[level_]), tmpMat, maxnnz, false, cputime);
// get a view from the graph
const Epetra_CrsGraph& graph = tmpMat->Graph();
// compare the graph to the existing one
same = compare_graphs(&graph,graph_);
destroyFD(); // we are here to recompute the FD-operator (this destroys graph_)
double t0 = GetClock();
graph_ = ML_NOX::deepcopy_graph(&graph);
// delete the copy of the Epetra_CrsMatrix
if (tmpMat) delete tmpMat; tmpMat = 0;
double t1 = GetClock();
if (ml_printlevel_ > 0 && 0 == comm_.MyPID())
cout << "matrixfreeML (level " << level_ << "): extraction/copy of Graph in " << cputime+t1-t0 << " sec\n"
<< " max-nonzeros in Graph: " << maxnnz << "\n";
}
// recreate this levels coarse interface
if (same)
coarseinterface_->recreate(ml_printlevel_,P,&(graph_->RowMap()));
else
{
delete coarseinterface_;
coarseinterface_ = new ML_NOX::Nox_CoarseProblem_Interface(fineinterface_,level_,ml_printlevel_,
P,&(graph_->RowMap()),nlevel_);
}
// restrict the xfine-vector to this level
Epetra_Vector* xthis = coarseinterface_->restrict_fine_to_this(xfine);
if (!xthis)
{
cout << "**ERR**: ML_Epetra::ML_Nox_MatrixfreeLevel::ML_Nox_MatrixfreeLevel:\n"
<< "**ERR**: ML_Epetra::Nox_CoarseProblem_Interface::restrict_fine_to_this returned NULL on level "
<< level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
Epetra_Vector* xc = new Epetra_Vector(graph_->RowMap(),false);
// FIXME: after intesive testing, this test might be obsolet
#if 0
bool samemap = xc->Map().PointSameAs(xthis->Map());
if (samemap)
{
#endif
xc->Update(1.0,*xthis,0.0);
#if 0
}
else
{
cout << "**WRN** Maps are not equal in\n"
<< "**WRN** file/line: " << __FILE__ << "/" << __LINE__ << "\n";
// import the xthis vector in the Map that ML produced for graph_
Epetra_Import* importer = new Epetra_Import(graph_->RowMap(),xthis->Map());
int ierr = xc->Import(*xthis,*importer,Insert);
if (ierr)
{
cout << "**ERR**: ML_Epetra::ML_Nox_MatrixfreeLevel::ML_Nox_MatrixfreeLevel:\n"
<< "**ERR**: export from xthis to xc returned err=" << ierr <<"\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
if (importer) delete importer; importer = 0;
}
#endif
if (xthis) delete xthis; xthis = 0;
// create the coloring of the graph
if (ml_printlevel_>0 && comm_.MyPID()==0)
{
cout << "matrixfreeML (level " << level_ << "): Entering Recoloring on level " << level_ << "\n";
fflush(stdout);
}
double t0 = GetClock();
if (!same) // te graph has obviously changed, so we need to recolor
{
if (colorMap_) delete colorMap_; colorMap_ = 0;
if (colorMapIndex_) delete colorMapIndex_; colorMapIndex_ = 0;
if (colorcolumns_) delete colorcolumns_; colorcolumns_ = 0;
colorMap_ = ML_NOX::ML_Nox_collapsedcoloring(graph_,bsize_,isDiagonalOnly_,ml_printlevel_);
if (!colorMap_) colorMap_ = ML_NOX::ML_Nox_standardcoloring(graph_,isDiagonalOnly_);
colorMapIndex_ = new EpetraExt::CrsGraph_MapColoringIndex(*colorMap_);
colorcolumns_ = &(*colorMapIndex_)(*graph_);
}
else if (ml_printlevel_>0 && comm_.MyPID()==0)
cout << "matrixfreeML (level " << level_ << "): Reusing existing Coloring on level " << level_ << "\n";
double t1 = GetClock();
if (ml_printlevel_>5)
{
cout << "matrixfreeML (level " << level_ << "): Proc " << comm_.MyPID() <<" (Re)Coloring time is " << (t1-t0) << " sec\n";
fflush(stdout);
}
#if 0
// print the colorMap_
if (comm_.MyPID()==0)
cout << "colorMap_\n";
cout << *colorMap_;
for (int i=0; i<colorcolumns_->size(); i++)
{
if (comm_.MyPID()==0)
cout << "the " << i << " th colorcolumn_ - vector\n";
cout << colorcolumns_->operator[](i);
}
#endif
// construct the FiniteDifferenceColoring-Matrix
if (ml_printlevel_>0 && comm_.MyPID()==0)
{
cout << "matrixfreeML (level " << level_ << "): Entering Construction FD-Operator on level " << level_ << "\n";
fflush(stdout);
}
t0 = GetClock();
#if 1 // FD-operator with coloring (see the #if 1 in ml_nox_matrixfreelevel.H as well!)
FD_ = new NOX::EpetraNew::FiniteDifferenceColoring(*coarseinterface_,
*xc,
*graph_,
*colorMap_,
*colorcolumns_,
true,
isDiagonalOnly_,
fd_beta_,fd_alpha_);
#else // FD-operator without coloring
FD_ = new NOX::EpetraNew::FiniteDifference(*coarseinterface_,
*xc,
*graph_,
fd_beta_,fd_alpha_);
#endif
// set differencing method
if (fd_centered_)
{
FD_->setDifferenceMethod(NOX::EpetraNew::FiniteDifferenceColoring::Centered);
}
bool err = FD_->computeJacobian(*xc);
if (err==false)
{
cout << "**ERR**: ML_Epetra::ML_Nox_MatrixfreeLevel::ML_Nox_MatrixfreeLevel:\n"
<< "**ERR**: NOX::Epetra::FiniteDifferenceColoring returned an error on level "
<< level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
t1 = GetClock();
if (ml_printlevel_>5)
cout << "matrixfreeML (level " << level_ << "): Proc " << comm_.MyPID()
<<" Finite Differencing operator constr. in " << (t1-t0) << " sec\n";
// get ref to computed Epetra_CrsMatrix
A_ = dynamic_cast<Epetra_CrsMatrix*>(&(FD_->getUnderlyingMatrix()));
// print number of calls to the coarse interface
if (ml_printlevel_>5 && comm_.MyPID()==0)
cout << "matrixfreeML (level " << level_ << "): Calls to coarse-computeF in FD-Operator: "
<< coarseinterface_->numcallscomputeF() << "\n";
// set counter for number of calls to the coarseinterface_->computeF back to zero
coarseinterface_->resetnumcallscomputeF();
// tidy up
if (xc) delete xc; xc = 0;
return true;
}
/*----------------------------------------------------------------------*
| Constructor (public) m.gee 01/05|
| IMPORTANT: |
| No matter on which level we are here, the vector xfine is ALWAYS |
| a fine grid vector here! |
| this is the constructor for the ismatrixfree==false case
*----------------------------------------------------------------------*/
ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel(
int level, int nlevel, int printlevel, ML* ml,
ML_Aggregate* ag,Epetra_CrsMatrix** P,
ML_NOX::Ml_Nox_Fineinterface& interface,
const Epetra_Comm& comm, const Epetra_Vector& xfine,
bool ismatrixfree, bool matfreelev0, bool isnlnCG,
int nitersCG, bool broyden, Epetra_CrsMatrix* Jac,
string fsmoothertype, string smoothertype,
string coarsesolvetype,
int nsmooth_fine, int nsmooth, int nsmooth_coarse,
double conv_normF, double conv_nupdate,
int conv_maxiter,int numPDE, int nullspdim)
: fineinterface_(interface),
comm_(comm)
{
level_ = level; // this level
nlevel_ = nlevel; // number of total levels
ml_printlevel_ = printlevel; // printlevel
ml_ = ml; // the global ML object
ag_ = ag; // the global ML_Aggregate object
thislevel_prec_ = 0; // this level's linear preconditioner
thislevel_ml_ = 0; // this level's local ML object
thislevel_ag_ = 0; // this level's local ML_Aggregate object
coarseinterface_ = 0; // this level's coarse interface
coarseprepost_ = 0;
xthis_ = 0; // this level's current solution matching this level's map!!!!
thislevel_A_ = 0; // this level's NOX Matrixfree operator
SmootherA_ = 0; // this level's Epetra_CrsMatrix for thislevel_prec_
ismatrixfree_ = ismatrixfree; // matrixfree flag
conv_normF_ = conv_normF; // NOX convergence test stuff
conv_nupdate_ = conv_nupdate;
conv_maxiter_ = conv_maxiter;
absresid_ = 0;
nupdate_ = 0;
fv_ = 0;
maxiters_ = 0;
combo1_ = 0;
combo2_ = 0;
thislevel_linSys_ = 0; // this level's NOX linear system
nlParams_ = 0; // NOX parameters
initialGuess_ = 0; // NOX initial guess
group_ = 0; // NOX group
solver_ = 0; // NOX solver
isnlnCG_ = isnlnCG;
azlinSys_ = 0;
clone_ = 0;
nitersCG_ = nitersCG;
broyden_ = broyden;
Broyd_ = 0;
if (ismatrixfree_==true)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ismatrixfree_==true on level " << level_ << "\n"
<< "**ERR**: in constructor for ismatrixfree_==false - case\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// ------------------------------------------------------------------------
// get the Jacobian of this level
const Epetra_CrsGraph* graph = 0;
// ------------------------------------------------------------------------
if (level_==0)
{
graph = fineinterface_.getGraph();
// On fine level this is the fineinterface's Jacobian
if (matfreelev0==false)
SmootherA_ = fineinterface_.getJacobian();
else if (matfreelev0==true && Jac)
SmootherA_ = Jac;
else
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: something weired happened\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
}
// ------------------------------------------------------------------------
else
{
// On coarse levels get Jacobian from hierarchy
// Note: On levels>0 SmootherA_ is a real copy of the Jacobian
int maxnnz=0;
double cputime=0.0;
ML_Operator2EpetraCrsMatrix(&(ml_->Amat[level_]), SmootherA_, maxnnz,
false, cputime);
SmootherA_->OptimizeStorage();
graph = &(SmootherA_->Graph());
}
// just to be save
if (!SmootherA_ || !graph)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: Smoother==NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// ------------------------------------------------------------------------
// generate this level's coarse interface
coarseinterface_ = new ML_NOX::Nox_CoarseProblem_Interface(
fineinterface_,level_,ml_printlevel_,
P,&(graph->RowMap()),nlevel_);
// ------------------------------------------------------------------------
// generate this level's coarse prepostoperator
if (level_==0)
coarseprepost_ = new ML_NOX::Ml_Nox_CoarsePrePostOperator(*coarseinterface_,
fineinterface_);
// ------------------------------------------------------------------------
// get the current solution to this level
xthis_ = coarseinterface_->restrict_fine_to_this(xfine);
// ------------------------------------------------------------------------
// create this level's preconditioner
// We use a 1-level ML-hierarchy for that
ML_Aggregate_Create(&thislevel_ag_);
ML_Create(&thislevel_ml_,1);
// set the Jacobian on level 0 of the local ml
EpetraMatrix2MLMatrix(thislevel_ml_,0,
(dynamic_cast<Epetra_RowMatrix*>(SmootherA_)));
// construct a 1-level ML-hierarchy on this level as a smoother
ML_Set_PrintLevel(ml_printlevel_);
ML_Aggregate_Set_CoarsenScheme_Uncoupled(thislevel_ag_);
ML_Aggregate_Set_DampingFactor(thislevel_ag_, 0.0);
ML_Aggregate_Set_Threshold(thislevel_ag_, 0.0);
ML_Aggregate_Set_MaxCoarseSize(thislevel_ag_,1);
ML_Aggregate_Set_NullSpace(thislevel_ag_,numPDE,nullspdim,NULL,
SmootherA_->NumMyRows());
int thislevel_nlevel = ML_Gen_MGHierarchy_UsingAggregation(thislevel_ml_,0,
ML_INCREASING,thislevel_ag_);
if (thislevel_nlevel != 1)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ML generated a local hierarchy of " << thislevel_nlevel << " on level " << level_ << "\n"
<< "**ERR**: this is supposed to be 1 Level only!\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// set the smoother
if (level_==0)
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,fsmoothertype,nsmooth_fine);
else if (level_ != nlevel_-1) // set the smoother from the input
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,smoothertype,nsmooth);
else // set the coarse solver from the input
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,coarsesolvetype,nsmooth_coarse);
// create this level's preconditioner class
ML_Epetra::MultiLevelOperator* ml_tmp = new ML_Epetra::MultiLevelOperator(
thislevel_ml_,comm_,
SmootherA_->OperatorDomainMap(),
SmootherA_->OperatorRangeMap());
thislevel_prec_ = new ML_NOX::ML_Nox_ConstrainedMultiLevelOperator(ml_tmp,*coarseinterface_);
if (!thislevel_prec_)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: thislevel_prec_==NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// ------------------------------------------------------------------------
// set up NOX on this level
// ------------------------------------------------------------------------
nlParams_ = new Teuchos::ParameterList();
Teuchos::ParameterList& printParams = nlParams_->sublist("Printing");
printParams.setParameter("MyPID", comm_.MyPID());
printParams.setParameter("Output Precision", 14);
printParams.setParameter("Output Processor", 0);
if (ml_printlevel_>9)
printParams.setParameter("Output Information",
NOX::Utils::OuterIteration +
NOX::Utils::Warning);
else if (ml_printlevel_>8)
printParams.setParameter("Output Information",
NOX::Utils::Warning);
else
printParams.setParameter("Output Information",0);
if (level_==0)
nlParams_->sublist("Solver Options").setParameter("User Defined Pre/Post Operator", *coarseprepost_);
nlParams_->setParameter("Nonlinear Solver", "Line Search Based");
Teuchos::ParameterList& searchParams = nlParams_->sublist("Line Search");
Teuchos::ParameterList* lsParamsptr = 0;
if (isnlnCG_)
{
searchParams.setParameter("Method", "NonlinearCG");
Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
dirParams.setParameter("Method", "NonlinearCG");
Teuchos::ParameterList& nlcgParams = dirParams.sublist("Nonlinear CG");
nlcgParams.setParameter("Restart Frequency", 10);
nlcgParams.setParameter("Precondition", "On");
nlcgParams.setParameter("Orthogonalize", "Polak-Ribiere");
//nlcgParams.setParameter("Orthogonalize", "Fletcher-Reeves");
Teuchos::ParameterList& lsParams = nlcgParams.sublist("Linear Solver");
lsParams.setParameter("Aztec Solver", "CG");
lsParams.setParameter("Max Iterations", 1);
lsParams.setParameter("Tolerance", 1e-11);
lsParams.setParameter("Output Frequency", 0);
lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
lsParams.setParameter("Preconditioner","User Defined");
}
else // Newton's method using ML-preconditioned Aztec as linear solver
{
searchParams.setParameter("Method", "Full Step");
// Sublist for direction
Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
dirParams.setParameter("Method", "Newton");
Teuchos::ParameterList& newtonParams = dirParams.sublist("Newton");
newtonParams.setParameter("Forcing Term Method", "Constant");
//newtonParams.setParameter("Forcing Term Method", "Type 1");
//newtonParams.setParameter("Forcing Term Method", "Type 2");
newtonParams.setParameter("Forcing Term Minimum Tolerance", 1.0e-6);
newtonParams.setParameter("Forcing Term Maximum Tolerance", 0.1);
Teuchos::ParameterList& lsParams = newtonParams.sublist("Linear Solver");
lsParamsptr = &lsParams;
lsParams.setParameter("Size of Krylov Subspace", 100);
lsParams.setParameter("Aztec Solver", "GMRES");
lsParams.setParameter("Max Iterations", nitersCG_);
lsParams.setParameter("Tolerance", conv_normF_); // FIXME? is this correct?
if (ml_printlevel_>8)
lsParams.setParameter("Output Frequency", 50);
else
lsParams.setParameter("Output Frequency", 0);
lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
lsParams.setParameter("Preconditioner","User Defined");
}
// create the initial guess
initialGuess_ = new NOX::Epetra::Vector(*xthis_, NOX::DeepCopy, true);
// NOTE: do not delete xthis_, it's used and destroyed by initialGuess_
// create the necessary interfaces
NOX::EpetraNew::Interface::Preconditioner* iPrec = 0;
NOX::EpetraNew::Interface::Required* iReq = 0;
NOX::EpetraNew::Interface::Jacobian* iJac = 0;
if (isnlnCG_)
{
// create the matrixfree operator used in the nlnCG
thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
// create the necessary interfaces
iPrec = 0;
iReq = coarseinterface_;
iJac = thislevel_A_;
// create the linear system
thislevel_linSys_ = new ML_NOX::Ml_Nox_LinearSystem(
*iJac,*thislevel_A_,*iPrec,
coarseinterface_,*thislevel_prec_,
*xthis_,ismatrixfree_,level_,ml_printlevel_);
// create the group
group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*thislevel_linSys_);
}
else // Modified Newton's method
{
if (!broyden_)
{
// create the necessary interfaces
iPrec = this;
iReq = coarseinterface_;
//iJac = this;
thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
// create the initial guess vector
//clone_ = new Epetra_Vector(*xthis_);
// create the linear system
//azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
// printParams,*lsParamsptr,
// *iJac,*SmootherA_,*iPrec,
// *thislevel_prec_,*clone_);
azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
printParams,*lsParamsptr,
*thislevel_A_,*thislevel_A_,*iPrec,
*thislevel_prec_,*xthis_);
}
else // use a Broyden update for the Jacobian
{
// create the initial guess vector
//clone_ = new Epetra_Vector(*xthis_);
// create the necessary interfaces
iPrec = this;
iReq = coarseinterface_;
Broyd_ = new NOX::EpetraNew::BroydenOperator(*nlParams_,*xthis_,
*SmootherA_,false);
// create the linear system
azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
printParams,*lsParamsptr,
*Broyd_,*SmootherA_,*iPrec,
*thislevel_prec_,*xthis_);
}
// create the group
group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,
*azlinSys_);
}
// create convergence test
create_Nox_Convergencetest(conv_normF_,conv_nupdate_,conv_maxiter_);
// create the solver
solver_ = new NOX::Solver::Manager(*group_,*combo2_,*nlParams_);
return;
}
int main(int argc, char *argv[])
{
int Nnodes=32*32; /* Total number of nodes in the problem.*/
/* 'Nnodes' must be a perfect square. */
struct user_partition Edge_Partition = {NULL, NULL,0,0,NULL,0,0,0},
Node_Partition = {NULL, NULL,0,0,NULL,0,0,0};
int proc_config[AZ_PROC_SIZE];
#ifdef ML_MPI
MPI_Init(&argc,&argv);
#endif
AZ_set_proc_config(proc_config, COMMUNICATOR);
ML_Comm* comm;
ML_Comm_Create(&comm);
Node_Partition.Nglobal = Nnodes;
Edge_Partition.Nglobal = Node_Partition.Nglobal*2;
user_partition_nodes(&Node_Partition);
user_partition_edges(&Edge_Partition, &Node_Partition);
AZ_MATRIX * AZ_Ke = user_Ke_build(&Edge_Partition);
AZ_MATRIX * AZ_Kn = user_Kn_build(&Node_Partition);
// convert (put wrappers) from Aztec matrices to ML_Operator's
ML_Operator * ML_Ke, * ML_Kn, * ML_Tmat;
ML_Ke = ML_Operator_Create( comm );
ML_Kn = ML_Operator_Create( comm );
AZ_convert_aztec_matrix_2ml_matrix(AZ_Ke,ML_Ke,proc_config);
AZ_convert_aztec_matrix_2ml_matrix(AZ_Kn,ML_Kn,proc_config);
ML_Tmat = user_T_build(&Edge_Partition, &Node_Partition,
ML_Kn, comm);
Epetra_CrsMatrix * Epetra_Kn, * Epetra_Ke, * Epetra_T;
int MaxNumNonzeros;
double CPUTime;
ML_Operator2EpetraCrsMatrix(ML_Ke,Epetra_Ke,
MaxNumNonzeros,
true,CPUTime);
ML_Operator2EpetraCrsMatrix(ML_Kn,
Epetra_Kn,MaxNumNonzeros,
true,CPUTime);
ML_Operator2EpetraCrsMatrix(ML_Tmat,Epetra_T,MaxNumNonzeros,
true,CPUTime);
Teuchos::ParameterList MLList;
ML_Epetra::SetDefaults("maxwell", MLList);
MLList.set("ML output", 0);
MLList.set("aggregation: type", "Uncoupled");
MLList.set("coarse: max size", 30);
MLList.set("aggregation: threshold", 0.0);
MLList.set("coarse: type", "Amesos-KLU");
ML_Epetra::MultiLevelPreconditioner * MLPrec =
new ML_Epetra::MultiLevelPreconditioner(*Epetra_Ke, *Epetra_T, *Epetra_Kn,
MLList);
Epetra_Vector LHS(Epetra_Ke->DomainMap()); LHS.Random();
Epetra_Vector RHS(Epetra_Ke->DomainMap()); RHS.PutScalar(1.0);
Epetra_LinearProblem Problem(Epetra_Ke,&LHS,&RHS);
AztecOO solver(Problem);
solver.SetPrecOperator(MLPrec);
solver.SetAztecOption(AZ_solver, AZ_cg_condnum);
solver.SetAztecOption(AZ_output, 32);
solver.Iterate(500, 1e-8);
// ========================= //
// compute the real residual //
// ========================= //
Epetra_Vector RHScomp(Epetra_Ke->DomainMap());
int ierr;
ierr = Epetra_Ke->Multiply(false, LHS, RHScomp);
assert(ierr==0);
Epetra_Vector resid(Epetra_Ke->DomainMap());
ierr = resid.Update(1.0, RHS, -1.0, RHScomp, 0.0);
assert(ierr==0);
double residual;
ierr = resid.Norm2(&residual);
assert(ierr==0);
if (proc_config[AZ_node] == 0) {
std::cout << std::endl;
std::cout << "==> Residual = " << residual << std::endl;
std::cout << std::endl;
}
// =============== //
// C L E A N U P //
// =============== //
delete MLPrec; // destroy phase prints out some information
delete Epetra_Kn;
delete Epetra_Ke;
delete Epetra_T;
ML_Operator_Destroy( &ML_Ke );
ML_Operator_Destroy( &ML_Kn );
ML_Comm_Destroy( &comm );
if (Edge_Partition.my_local_ids != NULL) free(Edge_Partition.my_local_ids);
if (Node_Partition.my_local_ids != NULL) free(Node_Partition.my_local_ids);
if (Node_Partition.my_global_ids != NULL) free(Node_Partition.my_global_ids);
if (Edge_Partition.my_global_ids != NULL) free(Edge_Partition.my_global_ids);
if (Node_Partition.needed_external_ids != NULL)
free(Node_Partition.needed_external_ids);
if (Edge_Partition.needed_external_ids != NULL)
free(Edge_Partition.needed_external_ids);
if (AZ_Ke!= NULL) {
AZ_free(AZ_Ke->bindx);
AZ_free(AZ_Ke->val);
AZ_free(AZ_Ke->data_org);
AZ_matrix_destroy(&AZ_Ke);
}
if (AZ_Kn!= NULL) {
AZ_free(AZ_Kn->bindx);
AZ_free(AZ_Kn->val);
AZ_free(AZ_Kn->data_org);
AZ_matrix_destroy(&AZ_Kn);
}
ML_Operator_Destroy(&ML_Tmat);
if (residual > 1e-5) {
std::cout << "`MultiLevelPreconditioner_Maxwell.exe' failed!" << std::endl;
exit(EXIT_FAILURE);
}
#ifdef ML_MPI
MPI_Finalize();
#endif
if (proc_config[AZ_node] == 0)
std::cout << "`MultiLevelPreconditioner_Maxwell.exe' passed!" << std::endl;
exit(EXIT_SUCCESS);
}
