int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
int mypid = Comm.MyPID();
#else
Epetra_SerialComm Comm;
int mypid = 0;
#endif
// Read XML input deck
ParameterList masterList;
if (argc > 1) {
if (strncmp("-h",argv[1],2) == 0) {
cout << "help" << endl;
ML_Print_Help();
ML_Exit(mypid,0,EXIT_SUCCESS);
}
else {
int i=0,j;
FILE* fid = fopen(argv[1],"r");
if (fid) {
i++;
fclose(fid);
}
Comm.SumAll(&i, &j, 1);
if (j!=Comm.NumProc()) {
cout << "Could not open input file." << endl;
ML_Print_Help();
ML_Exit(mypid,0,EXIT_FAILURE);
}
FileInputSource fileSrc(argv[1]);
XMLObject fileXML = fileSrc.getObject();
XMLParameterListReader ListReader;
masterList = ListReader.toParameterList(fileXML);
}
} else {
cout << "No input file specified." << endl;
ML_Print_Help();
ML_Exit(mypid,0,EXIT_SUCCESS);
}
ParameterList *fileList, *AztecOOList;
try {fileList = &(masterList.sublist("data files",true));}
catch(...) {ML_Exit(mypid,"Missing \"data files\" sublist.",EXIT_FAILURE);}
try {AztecOOList = &(masterList.sublist("AztecOO"));}
catch(...) {ML_Exit(mypid,"Missing \"AztecOO\" sublist.",EXIT_FAILURE);}
#ifdef ML_SCALING
const int ntimers=4;
enum {total, probBuild, precBuild, solve};
ml_DblLoc timeVec[ntimers], maxTime[ntimers], minTime[ntimers];
for (int i=0; i<ntimers; i++) timeVec[i].rank = Comm.MyPID();
timeVec[total].value = MPI_Wtime();
#endif
string matrixfile = fileList->get("matrix input file","A.dat");
const char *datafile = matrixfile.c_str();
int numGlobalRows;
ML_Read_Matrix_Dimensions(datafile, &numGlobalRows, Comm);
#ifdef ML_SCALING
timeVec[probBuild].value = MPI_Wtime();
#endif
// ===================================================== //
// READ IN MATRICES FROM FILE //
// ===================================================== //
if (!mypid) printf("reading %s\n",datafile); fflush(stdout);
Epetra_CrsMatrix *Amat=NULL;
//Epetra_Map *RowMap=NULL;
int errCode=0;
//if (RowMap) errCode=EpetraExt::MatrixMarketFileToCrsMatrix(datafile, *RowMap, Amat);
//else errCode=EpetraExt::MatrixMarketFileToCrsMatrix(datafile, Comm, Amat);
errCode=EpetraExt::MatrixMarketFileToCrsMatrix(datafile, Comm, Amat);
if (errCode) ML_Exit(mypid,"error reading matrix", EXIT_FAILURE);
Amat->OptimizeStorage();
Epetra_Vector LHS(Amat->RowMap()); LHS.Random();
Epetra_Vector RHS(Amat->RowMap()); RHS.PutScalar(0.0);
Epetra_LinearProblem Problem(Amat, &LHS, &RHS);
#ifdef ML_SCALING
timeVec[probBuild].value = MPI_Wtime() - timeVec[probBuild].value;
#endif
// =========================== build preconditioner ===========================
#ifdef ML_SCALING
timeVec[precBuild].value = MPI_Wtime();
#endif
// no preconditioner right now
#ifdef ML_SCALING
timeVec[precBuild].value = MPI_Wtime() - timeVec[precBuild].value;
#endif
// =========================== outer solver =============================
#ifdef ML_SCALING
timeVec[solve].value = MPI_Wtime();
#endif
solver.SetParameters(*AztecOOList);
int maxits = AztecOOList->get("Aztec iterations",250);
double tol = AztecOOList->get("Aztec tolerance",1e-10);
#ifdef ML_SCALING
timeVec[solve].value = MPI_Wtime() - timeVec[solve].value;
#endif
// compute the real residual
double residual;
LHS.Norm2(&residual);
if( Comm.MyPID()==0 ) {
cout << "||b-Ax||_2 = " << residual << endl;
}
delete Amat;
//delete RowMap;
#ifdef ML_SCALING
timeVec[total].value = MPI_Wtime() - timeVec[total].value;
//avg
double dupTime[ntimers],avgTime[ntimers];
for (int i=0; i<ntimers; i++) dupTime[i] = timeVec[i].value;
MPI_Reduce(dupTime,avgTime,ntimers,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
for (int i=0; i<ntimers; i++) avgTime[i] = avgTime[i]/Comm.NumProc();
//min
MPI_Reduce(timeVec,minTime,ntimers,MPI_DOUBLE_INT,MPI_MINLOC,0,MPI_COMM_WORLD);
//max
MPI_Reduce(timeVec,maxTime,ntimers,MPI_DOUBLE_INT,MPI_MAXLOC,0,MPI_COMM_WORLD);
if (Comm.MyPID() == 0) {
printf("timing : max (pid) min (pid) avg\n");
printf("Problem build : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[probBuild].value,maxTime[probBuild].rank,
minTime[probBuild].value,minTime[probBuild].rank,
avgTime[probBuild]);
printf("Preconditioner build : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[precBuild].value,maxTime[precBuild].rank,
minTime[precBuild].value,minTime[precBuild].rank,
avgTime[precBuild]);
printf("Solve : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[solve].value,maxTime[solve].rank,
minTime[solve].value,minTime[solve].rank,
avgTime[solve]);
printf("Total : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[total].value,maxTime[total].rank,
minTime[total].value,minTime[total].rank,
avgTime[total]);
}
#endif
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} //main
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
bool verbose = (Comm.MyPID() == 0);
double TotalResidual = 0.0;
// Create the Map, defined as a grid, of size nx x ny x nz,
// subdivided into mx x my x mz cubes, each assigned to a
// different processor.
#ifndef FILENAME_SPECIFIED_ON_COMMAND_LINE
ParameterList GaleriList;
GaleriList.set("nx", 4);
GaleriList.set("ny", 4);
GaleriList.set("nz", 4 * Comm.NumProc());
GaleriList.set("mx", 1);
GaleriList.set("my", 1);
GaleriList.set("mz", Comm.NumProc());
Epetra_Map* Map = CreateMap("Cartesian3D", Comm, GaleriList);
// Create a matrix, in this case corresponding to a 3D Laplacian
// discretized using a classical 7-point stencil. Please refer to
// the Galeri documentation for an overview of available matrices.
//
// NOTE: matrix must be symmetric if DSCPACK is used.
Epetra_CrsMatrix* Matrix = CreateCrsMatrix("Laplace3D", Map, GaleriList);
#else
bool transpose = false ;
bool distribute = false ;
bool symmetric ;
Epetra_CrsMatrix *Matrix = 0 ;
Epetra_Map *Map = 0 ;
MyCreateCrsMatrix( argv[1], Comm, Map, transpose, distribute, symmetric, Matrix ) ;
#endif
// build vectors, in this case with 1 vector
Epetra_MultiVector LHS(*Map, 1);
Epetra_MultiVector RHS(*Map, 1);
// create a linear problem object
Epetra_LinearProblem Problem(Matrix, &LHS, &RHS);
// use this list to set up parameters, now it is required
// to use all the available processes (if supported by the
// underlying solver). Uncomment the following two lines
// to let Amesos print out some timing and status information.
ParameterList List;
List.set("PrintTiming",true);
List.set("PrintStatus",true);
List.set("MaxProcs",Comm.NumProc());
std::vector<std::string> SolverType;
SolverType.push_back("Amesos_Paraklete");
SolverType.push_back("Amesos_Klu");
Comm.Barrier() ;
#if 1
SolverType.push_back("Amesos_Lapack");
SolverType.push_back("Amesos_Umfpack");
SolverType.push_back("Amesos_Pardiso");
SolverType.push_back("Amesos_Taucs");
SolverType.push_back("Amesos_Superlu");
SolverType.push_back("Amesos_Superludist");
SolverType.push_back("Amesos_Mumps");
SolverType.push_back("Amesos_Dscpack");
SolverType.push_back("Amesos_Scalapack");
#endif
Epetra_Time Time(Comm);
// this is the Amesos factory object that will create
// a specific Amesos solver.
Amesos Factory;
// Cycle over all solvers.
// Only installed solvers will be tested.
for (unsigned int i = 0 ; i < SolverType.size() ; ++i)
{
// Check whether the solver is available or not
if (Factory.Query(SolverType[i]))
{
// 1.- set exact solution (constant vector)
LHS.PutScalar(1.0);
// 2.- create corresponding rhs
Matrix->Multiply(false, LHS, RHS);
// 3.- randomize solution vector
LHS.Random();
// 4.- create the amesos solver object
Amesos_BaseSolver* Solver = Factory.Create(SolverType[i], Problem);
assert (Solver != 0);
Solver->SetParameters(List);
Solver->SetUseTranspose( true) ;
// 5.- factorize and solve
Comm.Barrier() ;
if (verbose)
std::cout << std::endl
<< "Solver " << SolverType[i]
<< ", verbose = " << verbose << std::endl ;
Comm.Barrier() ;
Time.ResetStartTime();
AMESOS_CHK_ERR(Solver->SymbolicFactorization());
if (verbose)
std::cout << std::endl
<< "Solver " << SolverType[i]
<< ", symbolic factorization time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
AMESOS_CHK_ERR(Solver->NumericFactorization());
if (verbose)
std::cout << "Solver " << SolverType[i]
<< ", numeric factorization time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
AMESOS_CHK_ERR(Solver->Solve());
if (verbose)
std::cout << "Solver " << SolverType[i]
<< ", solve time = "
<< Time.ElapsedTime() << std::endl;
Comm.Barrier() ;
// 6.- compute difference between exact solution and Amesos one
// (there are other ways of doing this in Epetra, but let's
// keep it simple)
double d = 0.0, d_tot = 0.0;
for (int j = 0 ; j< LHS.Map().NumMyElements() ; ++j)
d += (LHS[0][j] - 1.0) * (LHS[0][j] - 1.0);
Comm.SumAll(&d,&d_tot,1);
if (verbose)
std::cout << "Solver " << SolverType[i] << ", ||x - x_exact||_2 = "
<< sqrt(d_tot) << std::endl;
// 7.- delete the object
delete Solver;
TotalResidual += d_tot;
}
}
delete Matrix;
delete Map;
if (TotalResidual > 1e-9)
exit(EXIT_FAILURE);
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
} // end of main()
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
#define ML_SCALING
#ifdef ML_SCALING
const int ntimers=4;
enum {total, probBuild, precBuild, solve};
ml_DblLoc timeVec[ntimers], maxTime[ntimers], minTime[ntimers];
for (int i=0; i<ntimers; i++) timeVec[i].rank = Comm.MyPID();
timeVec[total].value = MPI_Wtime();
#endif
int nx;
if (argc > 1) nx = (int) strtol(argv[1],NULL,10);
else nx = 256;
if (nx < 1) nx = 256; // input a nonpositive integer if you want to specify
// the XML input file name.
nx = nx*(int)sqrt((double)Comm.NumProc());
int ny = nx;
printf("nx = %d\nny = %d\n",nx,ny);
fflush(stdout);
char xmlFile[80];
bool readXML=false;
if (argc > 2) {strcpy(xmlFile,argv[2]); readXML = true;}
else sprintf(xmlFile,"%s","params.xml");
ParameterList GaleriList;
GaleriList.set("nx", nx);
GaleriList.set("ny", ny);
#ifdef ML_SCALING
timeVec[probBuild].value = MPI_Wtime();
#endif
Epetra_Map* Map = CreateMap("Cartesian2D", Comm, GaleriList);
Epetra_CrsMatrix* A = CreateCrsMatrix("Laplace2D", Map, GaleriList);
if (!Comm.MyPID()) printf("nx = %d, ny = %d, mx = %d, my = %d\n",nx,ny,GaleriList.get("mx",-1),GaleriList.get("my",-1));
fflush(stdout);
//avoid potential overflow
double numMyRows = A->NumMyRows();
double numGlobalRows;
Comm.SumAll(&numMyRows,&numGlobalRows,1);
if (!Comm.MyPID()) printf("# global rows = %1.0f\n",numGlobalRows);
//printf("pid %d: #rows = %d\n",Comm.MyPID(),A->NumMyRows());
fflush(stdout);
Epetra_MultiVector *coords = CreateCartesianCoordinates("2D", Map,GaleriList);
double *x_coord=0,*y_coord=0,*z_coord=0;
double **ttt;
if (!coords->ExtractView(&ttt)) {
x_coord = ttt[0];
y_coord = ttt[1];
} else {
if (!Comm.MyPID()) printf("Error extracting coordinate vectors\n");
MPI_Finalize();
exit(EXIT_FAILURE);
}
Epetra_Vector LHS(*Map); LHS.Random();
Epetra_Vector RHS(*Map); RHS.PutScalar(0.0);
Epetra_LinearProblem Problem(A, &LHS, &RHS);
AztecOO solver(Problem);
#ifdef ML_SCALING
timeVec[probBuild].value = MPI_Wtime() - timeVec[probBuild].value;
#endif
// =========================== begin of ML part ===========================
#ifdef ML_SCALING
timeVec[precBuild].value = MPI_Wtime();
#endif
ParameterList MLList;
if (readXML) {
MLList.set("read XML",true);
MLList.set("XML input file",xmlFile);
}
else {
cout << "here" << endl;
ML_Epetra::SetDefaults("SA",MLList);
MLList.set("smoother: type","Chebyshev");
MLList.set("smoother: sweeps",3);
MLList.set("coarse: max size",1);
}
MLList.set("x-coordinates", x_coord);
MLList.set("y-coordinates", y_coord);
MLList.set("z-coordinates", z_coord);
/*
RCP<std::vector<int> >
m_smootherAztecOptions = rcp(new std::vector<int>(AZ_OPTIONS_SIZE));
RCP<std::vector<double> >
m_smootherAztecParams = rcp(new std::vector<double>(AZ_PARAMS_SIZE));
//int m_smootherAztecOptions[AZ_OPTIONS_SIZE];
//double m_smootherAztecParams[AZ_PARAMS_SIZE];
std::string smootherType("Aztec");
AZ_defaults(&(*m_smootherAztecOptions)[0],&(*m_smootherAztecParams)[0]);
(*m_smootherAztecOptions)[AZ_precond] = AZ_dom_decomp;
(*m_smootherAztecOptions)[AZ_subdomain_solve] = AZ_icc;
bool smootherAztecAsSolver = true;
*/
MLList.set("ML output",10);
MLList.set("repartition: enable",1);
MLList.set("repartition: max min ratio",1.3);
MLList.set("repartition: min per proc",200);
MLList.set("repartition: partitioner","Zoltan");
MLList.set("repartition: Zoltan dimensions",2);
MLList.set("repartition: put on single proc",1);
MLList.set("repartition: Zoltan type","hypergraph");
MLList.set("repartition: estimated iterations",13);
/*
MLList.set("smoother: Aztec options",m_smootherAztecOptions);
MLList.set("smoother: Aztec params",m_smootherAztecParams);
MLList.set("smoother: type",smootherType.c_str());
MLList.set("smoother: Aztec as solver", smootherAztecAsSolver);
MLList.set("ML print initial list", 0);
MLList.set("ML print final list", 0);
*/
ML_Epetra::MultiLevelPreconditioner* MLPrec =
new ML_Epetra::MultiLevelPreconditioner(*A, MLList);
// verify unused parameters on process 0 (put -1 to print on all
// processes)
MLPrec->PrintUnused(0);
#ifdef ML_SCALING
timeVec[precBuild].value = MPI_Wtime() - timeVec[precBuild].value;
#endif
// =========================== end of ML part =============================
#ifdef ML_SCALING
timeVec[solve].value = MPI_Wtime();
#endif
solver.SetPrecOperator(MLPrec);
solver.SetAztecOption(AZ_solver, AZ_cg);
solver.SetAztecOption(AZ_output, 1);
solver.Iterate(500, 1e-12);
#ifdef ML_SCALING
timeVec[solve].value = MPI_Wtime() - timeVec[solve].value;
#endif
// destroy the preconditioner
delete MLPrec;
// compute the real residual
double residual;
LHS.Norm2(&residual);
if( Comm.MyPID()==0 ) {
cout << "||b-Ax||_2 = " << residual << endl;
}
// for testing purposes
if (residual > 1e-5)
exit(EXIT_FAILURE);
delete A;
delete Map;
delete coords;
#ifdef ML_SCALING
timeVec[total].value = MPI_Wtime() - timeVec[total].value;
//avg
double dupTime[ntimers],avgTime[ntimers];
for (int i=0; i<ntimers; i++) dupTime[i] = timeVec[i].value;
MPI_Reduce(dupTime,avgTime,ntimers,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
for (int i=0; i<ntimers; i++) avgTime[i] = avgTime[i]/Comm.NumProc();
//min
MPI_Reduce(timeVec,minTime,ntimers,MPI_DOUBLE_INT,MPI_MINLOC,0,MPI_COMM_WORLD);
//max
MPI_Reduce(timeVec,maxTime,ntimers,MPI_DOUBLE_INT,MPI_MAXLOC,0,MPI_COMM_WORLD);
if (Comm.MyPID() == 0) {
printf("timing : max (pid) min (pid) avg\n");
printf("Problem build : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[probBuild].value,maxTime[probBuild].rank,
minTime[probBuild].value,minTime[probBuild].rank,
avgTime[probBuild]);
printf("Preconditioner build : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[precBuild].value,maxTime[precBuild].rank,
minTime[precBuild].value,minTime[precBuild].rank,
avgTime[precBuild]);
printf("Solve : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[solve].value,maxTime[solve].rank,
minTime[solve].value,minTime[solve].rank,
avgTime[solve]);
printf("Total : %2.3e (%d) %2.3e (%d) %2.3e \n",
maxTime[total].value,maxTime[total].rank,
minTime[total].value,minTime[total].rank,
avgTime[total]);
}
#endif
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return(EXIT_SUCCESS);
}
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc, &argv);
// define an Epetra communicator
Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
Epetra_SerialComm Comm;
#endif
// get the proc ID of this process
int MyPID = Comm.MyPID();
// get the total number of processes
int NumProc = Comm.NumProc();
// output some information to std output
cout << Comm << endl;
// ======================== //
// now some basic MPI calls //
// ------------------------ //
int ivalue;
double dvalue, dvalue2;
double* dvalues; dvalues = new double[NumProc];
double* dvalues2; dvalues2 = new double[NumProc];
int root = 0;
// equivalent to MPI_Barrier
Comm.Barrier();
if (MyPID == root) dvalue = 12.0;
// On input, the root processor contains the list of values
// (in this case, a single value). On exit, all processes will
// have he same list of values. Note that all values must be allocated
// vefore the broadcast
// equivalent to MPI_Broadcast
Comm.Broadcast(&dvalue, 1, root);
// as before, but with integer values. As C++ can bind to the appropriate
// interface based on argument typing, the type of data is not required.
Comm.Broadcast(&ivalue, 1, root);
// equivalent MPI_Allgather
Comm.GatherAll(dvalues, dvalues2, 1);
// equivalent to MPI_Allreduce with MPI_SUM
dvalue = 1.0*MyPID;
Comm.SumAll( &dvalue, dvalues, 1);
// equivalent to MPI_Allreduce with MPI_SUM
Comm.MaxAll( &dvalue, dvalues, 1);
// equiavant to MPI_Scan with MPI_SUM
dvalue = 1.0 * MyPID;
Comm.ScanSum(&dvalue, &dvalue2, 1);
cout << "On proc " << MyPID << " dvalue2 = " << dvalue2 << endl;
delete[] dvalues;
delete[] dvalues2;
// ======================= //
// Finalize MPI and return //
// ----------------------- //
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return( EXIT_SUCCESS );
} /* main */
