double AztecOO_StatusTestResNorm::ComputeNorm(const Epetra_Vector & vec, NormType typeofnorm) {
double result = 0.0;
if (typeofnorm==TwoNorm) vec.Norm2(&result);
else if (typeofnorm==OneNorm) vec.Norm1(&result);
else vec.NormInf(&result);
return(result);
}
// ================================================ ====== ==== ==== == =
// the tentative null space is in input because the user
// has to remember to allocate and fill it, and then to delete
// it after calling this method.
int ML_Epetra::MultiLevelPreconditioner::
ComputeAdaptivePreconditioner(int TentativeNullSpaceSize,
double* TentativeNullSpace)
{
if ((TentativeNullSpaceSize == 0) || (TentativeNullSpace == 0))
ML_CHK_ERR(-1);
// ================================== //
// get parameters from the input list //
// ================================== //
// maximum number of relaxation sweeps
int MaxSweeps = List_.get("adaptive: max sweeps", 10);
// number of std::vector to be added to the tentative null space
int NumAdaptiveVectors = List_.get("adaptive: num vectors", 1);
if (verbose_) {
std::cout << PrintMsg_ << "*** Adaptive Smoother Aggregation setup ***" << std::endl;
std::cout << PrintMsg_ << " Maximum relaxation sweeps = " << MaxSweeps << std::endl;
std::cout << PrintMsg_ << " Additional vectors to compute = " << NumAdaptiveVectors << std::endl;
}
// ==================================================== //
// compute the preconditioner, set null space from user //
// (who will have to delete std::vector TentativeNullSpace) //
// ==================================================== //
double* NewNullSpace = 0;
double* OldNullSpace = TentativeNullSpace;
int OldNullSpaceSize = TentativeNullSpaceSize;
// need some work otherwise matvec() with Epetra_Vbr fails.
// Also, don't differentiate between range and domain here
// as ML will not work if range != domain
const Epetra_VbrMatrix* VbrA = NULL;
VbrA = dynamic_cast<const Epetra_VbrMatrix*>(RowMatrix_);
Epetra_Vector* LHS = 0;
Epetra_Vector* RHS = 0;
if (VbrA != 0) {
LHS = new Epetra_Vector(VbrA->DomainMap());
RHS = new Epetra_Vector(VbrA->DomainMap());
} else {
LHS = new Epetra_Vector(RowMatrix_->OperatorDomainMap());
RHS = new Epetra_Vector(RowMatrix_->OperatorDomainMap());
}
// destroy what we may already have
if (IsComputePreconditionerOK_ == true) {
DestroyPreconditioner();
}
// build the preconditioner for the first time
List_.set("null space: type", "pre-computed");
List_.set("null space: dimension", OldNullSpaceSize);
List_.set("null space: vectors", OldNullSpace);
ComputePreconditioner();
// ====================== //
// add one std::vector at time //
// ====================== //
for (int istep = 0 ; istep < NumAdaptiveVectors ; ++istep) {
if (verbose_) {
std::cout << PrintMsg_ << "\tAdaptation step " << istep << std::endl;
std::cout << PrintMsg_ << "\t---------------" << std::endl;
}
// ==================== //
// look for "bad" modes //
// ==================== //
// note: should an error occur, ML_CHK_ERR will return,
// and LHS and RHS will *not* be delete'd (--> memory leak).
// Anyway, this means that something wrong happened in the code
// and should be fixed by the user.
LHS->Random();
double Norm2;
for (int i = 0 ; i < MaxSweeps ; ++i) {
// RHS = (I - ML^{-1} A) LHS
ML_CHK_ERR(RowMatrix_->Multiply(false,*LHS,*RHS));
// FIXME: can do something slightly better here
ML_CHK_ERR(ApplyInverse(*RHS,*RHS));
ML_CHK_ERR(LHS->Update(-1.0,*RHS,1.0));
LHS->Norm2(&Norm2);
if (verbose_) {
std::cout << PrintMsg_ << "\titer " << i << ", ||x||_2 = ";
std::cout << Norm2 << std::endl;
}
}
// scaling vectors
double NormInf;
LHS->NormInf(&NormInf);
LHS->Scale(1.0 / NormInf);
// ========================================================= //
// copy tentative and computed null space into NewNullSpace, //
// which now becomes the standard null space //
// ========================================================= //
int NewNullSpaceSize = OldNullSpaceSize + 1;
NewNullSpace = new double[NumMyRows() * NewNullSpaceSize];
assert (NewNullSpace != 0);
int itmp = OldNullSpaceSize * NumMyRows();
for (int i = 0 ; i < itmp ; ++i) {
NewNullSpace[i] = OldNullSpace[i];
}
for (int j = 0 ; j < NumMyRows() ; ++j) {
NewNullSpace[itmp + j] = (*LHS)[j];
}
// =============== //
// visualize modes //
// =============== //
if (List_.get("adaptive: visualize", false)) {
double* x_coord = List_.get("viz: x-coordinates", (double*)0);
double* y_coord = List_.get("viz: y-coordinates", (double*)0);
double* z_coord = List_.get("viz: z-coordinates", (double*)0);
assert (x_coord != 0);
std::vector<double> plot_me(NumMyRows()/NumPDEEqns_);
ML_Aggregate_Viz_Stats info;
info.Amatrix = &(ml_->Amat[LevelID_[0]]);
info.x = x_coord;
info.y = y_coord;
info.z = z_coord;
info.Nlocal = NumMyRows() / NumPDEEqns_;
info.Naggregates = 1;
ML_Operator_AmalgamateAndDropWeak(&(ml_->Amat[LevelID_[0]]),
NumPDEEqns_, 0.0);
for (int ieqn = 0 ; ieqn < NumPDEEqns_ ; ++ieqn) {
for (int j = 0 ; j < NumMyRows() ; j+=NumPDEEqns_) {
plot_me[j / NumPDEEqns_] = (*LHS)[j + ieqn];
}
char FileName[80];
sprintf(FileName,"nullspace-mode%d-eq%d.xyz", istep, ieqn);
if (verbose_)
std::cout << PrintMsg_ << "writing file " << FileName << "..." << std::endl;
ML_Aggregate_VisualizeXYZ(info,FileName,
ml_->comm,&plot_me[0]);
}
ML_Operator_UnAmalgamateAndDropWeak(&(ml_->Amat[LevelID_[0]]),
NumPDEEqns_, 0.0);
}
// Destroy the old preconditioner
DestroyPreconditioner();
// ==================================================== //
// build the new preconditioner with the new null space //
// ==================================================== //
List_.set("null space: type", "pre-computed");
List_.set("null space: dimension", NewNullSpaceSize);
List_.set("null space: vectors", NewNullSpace);
ML_CHK_ERR(ComputePreconditioner());
if (istep && (istep != NumAdaptiveVectors))
delete OldNullSpace;
OldNullSpace = NewNullSpace;
OldNullSpaceSize = NewNullSpaceSize;
}
// keep trace of this pointer, it will be delete'd later
NullSpaceToFree_ = NewNullSpace;
delete LHS;
delete RHS;
return(0);
}
int main(int argc, char *argv[]) {
#ifdef EPETRA_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm (MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
cout << Comm << endl;
int MyPID = Comm.MyPID();
bool verbose = false;
bool verbose1 = true;
if (MyPID==0) verbose = true;
if(argc < 2 && verbose) {
cerr << "Usage: " << argv[0]
<< " HB_filename [level_fill [level_overlap [absolute_threshold [ relative_threshold]]]]" << endl
<< "where:" << endl
<< "HB_filename - filename and path of a Harwell-Boeing data set" << endl
<< "level_fill - The amount of fill to use for ILU(k) preconditioner (default 0)" << endl
<< "level_overlap - The amount of overlap used for overlapping Schwarz subdomains (default 0)" << endl
<< "absolute_threshold - The minimum value to place on the diagonal prior to factorization (default 0.0)" << endl
<< "relative_threshold - The relative amount to perturb the diagonal prior to factorization (default 1.0)" << endl << endl
<< "To specify a non-default value for one of these parameters, you must specify all" << endl
<< " preceding values but not any subsequent parameters. Example:" << endl
<< "ifpackHpcSerialMsr.exe mymatrix.hpc 1 - loads mymatrix.hpc, uses level fill of one, all other values are defaults" << endl
<< endl;
return(1);
}
// Uncomment the next three lines to debug in mpi mode
//int tmp;
//if (MyPID==0) cin >> tmp;
//Comm.Barrier();
Epetra_Map * readMap;
Epetra_CrsMatrix * readA;
Epetra_Vector * readx;
Epetra_Vector * readb;
Epetra_Vector * readxexact;
// Call routine to read in HB problem
Trilinos_Util_ReadHb2Epetra(argv[1], Comm, readMap, readA, readx, readb, readxexact);
// Create uniform distributed map
Epetra_Map map(readMap->NumGlobalElements(), 0, Comm);
// Create Exporter to distribute read-in matrix and vectors
Epetra_Export exporter(*readMap, map);
Epetra_CrsMatrix A(Copy, map, 0);
Epetra_Vector x(map);
Epetra_Vector b(map);
Epetra_Vector xexact(map);
Epetra_Time FillTimer(Comm);
x.Export(*readx, exporter, Add);
b.Export(*readb, exporter, Add);
xexact.Export(*readxexact, exporter, Add);
Comm.Barrier();
double vectorRedistributeTime = FillTimer.ElapsedTime();
A.Export(*readA, exporter, Add);
Comm.Barrier();
double matrixRedistributeTime = FillTimer.ElapsedTime() - vectorRedistributeTime;
assert(A.FillComplete()==0);
Comm.Barrier();
double fillCompleteTime = FillTimer.ElapsedTime() - matrixRedistributeTime;
if (Comm.MyPID()==0) {
cout << "\n\n****************************************************" << endl;
cout << "\n Vector redistribute time (sec) = " << vectorRedistributeTime<< endl;
cout << " Matrix redistribute time (sec) = " << matrixRedistributeTime << endl;
cout << " Transform to Local time (sec) = " << fillCompleteTime << endl<< endl;
}
Epetra_Vector tmp1(*readMap);
Epetra_Vector tmp2(map);
readA->Multiply(false, *readxexact, tmp1);
A.Multiply(false, xexact, tmp2);
double residual;
tmp1.Norm2(&residual);
if (verbose) cout << "Norm of Ax from file = " << residual << endl;
tmp2.Norm2(&residual);
if (verbose) cout << "Norm of Ax after redistribution = " << residual << endl << endl << endl;
//cout << "A from file = " << *readA << endl << endl << endl;
//cout << "A after dist = " << A << endl << endl << endl;
delete readA;
delete readx;
delete readb;
delete readxexact;
delete readMap;
Comm.Barrier();
bool smallProblem = false;
if (A.RowMap().NumGlobalElements()<100) smallProblem = true;
if (smallProblem)
cout << "Original Matrix = " << endl << A << endl;
x.PutScalar(0.0);
Epetra_LinearProblem FullProblem(&A, &x, &b);
double normb, norma;
b.NormInf(&normb);
norma = A.NormInf();
if (verbose)
cout << "Inf norm of Original Matrix = " << norma << endl
<< "Inf norm of Original RHS = " << normb << endl;
Epetra_Time ReductionTimer(Comm);
Epetra_CrsSingletonFilter SingletonFilter;
Comm.Barrier();
double reduceInitTime = ReductionTimer.ElapsedTime();
SingletonFilter.Analyze(&A);
Comm.Barrier();
double reduceAnalyzeTime = ReductionTimer.ElapsedTime() - reduceInitTime;
if (SingletonFilter.SingletonsDetected())
cout << "Singletons found" << endl;
else {
cout << "Singletons not found" << endl;
exit(1);
}
SingletonFilter.ConstructReducedProblem(&FullProblem);
Comm.Barrier();
double reduceConstructTime = ReductionTimer.ElapsedTime() - reduceInitTime;
double totalReduceTime = ReductionTimer.ElapsedTime();
if (verbose)
cout << "\n\n****************************************************" << endl
<< " Reduction init time (sec) = " << reduceInitTime<< endl
<< " Reduction Analyze time (sec) = " << reduceAnalyzeTime << endl
<< " Construct Reduced Problem time (sec) = " << reduceConstructTime << endl
<< " Reduction Total time (sec) = " << totalReduceTime << endl<< endl;
Statistics(SingletonFilter);
Epetra_LinearProblem * ReducedProblem = SingletonFilter.ReducedProblem();
Epetra_CrsMatrix * Ap = dynamic_cast<Epetra_CrsMatrix *>(ReducedProblem->GetMatrix());
Epetra_Vector * bp = (*ReducedProblem->GetRHS())(0);
Epetra_Vector * xp = (*ReducedProblem->GetLHS())(0);
if (smallProblem)
cout << " Reduced Matrix = " << endl << *Ap << endl
<< " LHS before sol = " << endl << *xp << endl
<< " RHS = " << endl << *bp << endl;
// Construct ILU preconditioner
double elapsed_time, total_flops, MFLOPs;
Epetra_Time timer(Comm);
int LevelFill = 0;
if (argc > 2) LevelFill = atoi(argv[2]);
if (verbose) cout << "Using Level Fill = " << LevelFill << endl;
int Overlap = 0;
if (argc > 3) Overlap = atoi(argv[3]);
if (verbose) cout << "Using Level Overlap = " << Overlap << endl;
double Athresh = 0.0;
if (argc > 4) Athresh = atof(argv[4]);
if (verbose) cout << "Using Absolute Threshold Value of = " << Athresh << endl;
double Rthresh = 1.0;
if (argc > 5) Rthresh = atof(argv[5]);
if (verbose) cout << "Using Relative Threshold Value of = " << Rthresh << endl;
Ifpack_IlukGraph * IlukGraph = 0;
Ifpack_CrsRiluk * ILUK = 0;
if (LevelFill>-1) {
elapsed_time = timer.ElapsedTime();
IlukGraph = new Ifpack_IlukGraph(Ap->Graph(), LevelFill, Overlap);
assert(IlukGraph->ConstructFilledGraph()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
if (verbose) cout << "Time to construct ILUK graph = " << elapsed_time << endl;
Epetra_Flops fact_counter;
elapsed_time = timer.ElapsedTime();
ILUK = new Ifpack_CrsRiluk(*IlukGraph);
ILUK->SetFlopCounter(fact_counter);
ILUK->SetAbsoluteThreshold(Athresh);
ILUK->SetRelativeThreshold(Rthresh);
//assert(ILUK->InitValues()==0);
int initerr = ILUK->InitValues(*Ap);
if (initerr!=0) {
cout << endl << Comm << endl << " InitValues error = " << initerr;
if (initerr==1) cout << " Zero diagonal found, warning error only";
cout << endl << endl;
}
assert(ILUK->Factor()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = ILUK->Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute preconditioner values = "
<< elapsed_time << endl
<< "MFLOPS for Factorization = " << MFLOPs << endl;
//cout << *ILUK << endl;
double Condest;
ILUK->Condest(false, Condest);
if (verbose) cout << "Condition number estimate for this preconditioner = " << Condest << endl;
}
int Maxiter = 100;
double Tolerance = 1.0E-8;
Epetra_Flops counter;
Ap->SetFlopCounter(counter);
xp->SetFlopCounter(*Ap);
bp->SetFlopCounter(*Ap);
if (ILUK!=0) ILUK->SetFlopCounter(*Ap);
elapsed_time = timer.ElapsedTime();
double normreducedb, normreduceda;
bp->NormInf(&normreducedb);
normreduceda = Ap->NormInf();
if (verbose)
cout << "Inf norm of Reduced Matrix = " << normreduceda << endl
<< "Inf norm of Reduced RHS = " << normreducedb << endl;
BiCGSTAB(*Ap, *xp, *bp, ILUK, Maxiter, Tolerance, &residual, verbose);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = counter.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute solution = "
<< elapsed_time << endl
<< "Number of operations in solve = " << total_flops << endl
<< "MFLOPS for Solve = " << MFLOPs<< endl << endl;
SingletonFilter.ComputeFullSolution();
if (smallProblem)
cout << " Reduced LHS after sol = " << endl << *xp << endl
<< " Full LHS after sol = " << endl << x << endl
<< " Full Exact LHS = " << endl << xexact << endl;
Epetra_Vector resid(x);
resid.Update(1.0, x, -1.0, xexact, 0.0); // resid = xcomp - xexact
resid.Norm2(&residual);
double normx, normxexact;
x.Norm2(&normx);
xexact.Norm2(&normxexact);
if (verbose)
cout << "2-norm of computed solution = " << normx << endl
<< "2-norm of exact solution = " << normxexact << endl
<< "2-norm of difference between computed and exact solution = " << residual << endl;
if (verbose1 && residual>1.0e-5) {
if (verbose)
cout << "Difference between computed and exact solution appears large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
Epetra_Vector bdiff(b);
assert(A.Multiply(false, resid, bdiff)==0);
assert(bdiff.Norm2(&residual)==0);
if (verbose)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
if (verbose)
cout << "********************************************************" << endl
<< " Solving again with 2*Ax=2*b" << endl
<< "********************************************************" << endl;
A.Scale(1.0); // A = 2*A
b.Scale(1.0); // b = 2*b
x.PutScalar(0.0);
b.NormInf(&normb);
norma = A.NormInf();
if (verbose)
cout << "Inf norm of Original Matrix = " << norma << endl
<< "Inf norm of Original RHS = " << normb << endl;
double updateReducedProblemTime = ReductionTimer.ElapsedTime();
SingletonFilter.UpdateReducedProblem(&FullProblem);
Comm.Barrier();
updateReducedProblemTime = ReductionTimer.ElapsedTime() - updateReducedProblemTime;
if (verbose)
cout << "\n\n****************************************************" << endl
<< " Update Reduced Problem time (sec) = " << updateReducedProblemTime<< endl
<< "****************************************************" << endl;
Statistics(SingletonFilter);
if (LevelFill>-1) {
Epetra_Flops fact_counter;
elapsed_time = timer.ElapsedTime();
int initerr = ILUK->InitValues(*Ap);
if (initerr!=0) {
cout << endl << Comm << endl << " InitValues error = " << initerr;
if (initerr==1) cout << " Zero diagonal found, warning error only";
cout << endl << endl;
}
assert(ILUK->Factor()==0);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = ILUK->Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute preconditioner values = "
<< elapsed_time << endl
<< "MFLOPS for Factorization = " << MFLOPs << endl;
double Condest;
ILUK->Condest(false, Condest);
if (verbose) cout << "Condition number estimate for this preconditioner = " << Condest << endl;
}
bp->NormInf(&normreducedb);
normreduceda = Ap->NormInf();
if (verbose)
cout << "Inf norm of Reduced Matrix = " << normreduceda << endl
<< "Inf norm of Reduced RHS = " << normreducedb << endl;
BiCGSTAB(*Ap, *xp, *bp, ILUK, Maxiter, Tolerance, &residual, verbose);
elapsed_time = timer.ElapsedTime() - elapsed_time;
total_flops = counter.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose) cout << "Time to compute solution = "
<< elapsed_time << endl
<< "Number of operations in solve = " << total_flops << endl
<< "MFLOPS for Solve = " << MFLOPs<< endl << endl;
SingletonFilter.ComputeFullSolution();
if (smallProblem)
cout << " Reduced LHS after sol = " << endl << *xp << endl
<< " Full LHS after sol = " << endl << x << endl
<< " Full Exact LHS = " << endl << xexact << endl;
resid.Update(1.0, x, -1.0, xexact, 0.0); // resid = xcomp - xexact
resid.Norm2(&residual);
x.Norm2(&normx);
xexact.Norm2(&normxexact);
if (verbose)
cout << "2-norm of computed solution = " << normx << endl
<< "2-norm of exact solution = " << normxexact << endl
<< "2-norm of difference between computed and exact solution = " << residual << endl;
if (verbose1 && residual>1.0e-5) {
if (verbose)
cout << "Difference between computed and exact solution appears large..." << endl
<< "Computing norm of A times this difference. If this norm is small, then matrix is singular"
<< endl;
Epetra_Vector bdiff(b);
assert(A.Multiply(false, resid, bdiff)==0);
assert(bdiff.Norm2(&residual)==0);
if (verbose)
cout << "2-norm of A times difference between computed and exact solution = " << residual << endl;
}
if (ILUK!=0) delete ILUK;
if (IlukGraph!=0) delete IlukGraph;
#ifdef EPETRA_MPI
MPI_Finalize() ;
#endif
return 0 ;
}