// ======================================================================
void Finalize()
{
if (ML_Comm_) {
ML_Comm_Destroy(&ML_Comm_);
ML_Comm_ = 0;
}
if (Epetra_Comm_) {
delete Epetra_Comm_;
Epetra_Comm_ = 0;
}
}
// ================================================ ====== ==== ==== == =
// Destroys all structures allocated in \c ComputePreconditioner() if the preconditioner has been computed.
int ML_Epetra::FaceMatrixFreePreconditioner::DestroyPreconditioner(){
if (ml_comm_) { ML_Comm_Destroy(&ml_comm_); ml_comm_ = 0; }// will need this
if (Prolongator_) {delete Prolongator_; Prolongator_=0;}
if (InvDiagonal_) {delete InvDiagonal_; InvDiagonal_=0;}
if (CoarsePC) {delete CoarsePC; CoarsePC=0;}
if (CoarseMatrix) {delete CoarseMatrix; CoarseMatrix=0;}
if (CoarseMat_ML) {ML_Operator_Destroy(&CoarseMat_ML);CoarseMat_ML=0;}
if (CoarseMap_) {delete CoarseMap_; CoarseMap_=0;}
#ifdef HAVE_ML_IFPACK
if (Smoother_){delete Smoother_; Smoother_=0;}
#endif
return 0;
}/*end DestroyPreconditioner*/
// ================================================ ====== ==== ==== == =
// Computes C= <me> * A
int ML_Epetra::ML_RefMaxwell_11_Operator::MatrixMatrix_Multiply(const Epetra_CrsMatrix & A, Epetra_CrsMatrix **C) const
{
ML_Comm* comm;
ML_Comm_Create(&comm);
#ifdef ML_MPI
const Epetra_MpiComm *epcomm = dynamic_cast<const Epetra_MpiComm*>(&(A.Comm()));
// Get the MPI communicator, as it may not be MPI_COMM_W0RLD, and update the ML comm object
if (epcomm) ML_Comm_Set_UsrComm(comm,epcomm->Comm());
#endif
ML_Operator *C_;
int rv=MatrixMatrix_Multiply(A,comm,&C_);
Epetra_CrsMatrix_Wrap_ML_Operator(C_,*Comm_,*DomainMap_,C,Copy,A.IndexBase());
ML_Operator_Destroy(&C_);
ML_Comm_Destroy(&comm);
return rv;
}/*end MatrixMatrix_Multiply*/
int main(int argc, char *argv[])
{
#ifdef ML_MPI
MPI_Init(&argc,&argv);
// This next bit of code drops a middle rank out of the calculation. This tests
// that the Hiptmair smoother does not hang in its apply. Hiptmair creates
// two separate ML objects, one for edge and one for nodes. By default, the ML
// objects use MPI_COMM_WORLD, whereas the matrix that Hiptmair is being applied to
// may have an MPI subcommunicator.
int commWorldSize;
MPI_Comm_size(MPI_COMM_WORLD,&commWorldSize);
std::vector<int> splitKey;
int rankToDrop = 1;
for (int i=0; i<commWorldSize; ++i)
splitKey.push_back(0);
splitKey[rankToDrop] = MPI_UNDEFINED; //drop the last process from subcommunicator
int myrank;
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
MPI_Comm subcomm;
MPI_Comm_split(MPI_COMM_WORLD, splitKey[myrank], myrank, &subcomm);
if (myrank == rankToDrop) goto droppedRankLabel;
#endif
{ //scoping to avoid compiler error about goto jumping over initialization
#ifdef ML_MPI
Epetra_MpiComm Comm(subcomm);
#else
Epetra_SerialComm Comm;
#endif
ML_Comm_Create(&mlcomm);
char *datafile;
#ifdef CurlCurlAndMassAreSeparate
if (argc != 5 && argc != 7) {
if (Comm.MyPID() == 0) {
std::cout << "usage: ml_maxwell.exe <S> <M> <T> <Kn> [edge map] [node map]"
<< std::endl;
std::cout << " S = edge stiffness matrix file" << std::endl;
std::cout << " M = edge mass matrix file" << std::endl;
std::cout << " T = discrete gradient file" << std::endl;
std::cout << " Kn = auxiliary nodal FE matrix file" << std::endl;
std::cout << " edge map = edge distribution over processors" << std::endl;
std::cout << " node map = node distribution over processors" << std::endl;
std::cout << argc << std::endl;
}
#else //ifdef CurlCurlAndMassAreSeparate
if (argc != 4 && argc != 6) {
if (Comm.MyPID() == 0) {
std::cout << "usage: ml_maxwell.exe <A> <T> <Kn> [edge map] [node map]" <<std::endl;
std::cout << " A = edge element matrix file" << std::endl;
std::cout << " T = discrete gradient file" << std::endl;
std::cout << " Kn = auxiliary nodal FE matrix file" << std::endl;
std::cout << " edge map = edge distribution over processors" << std::endl;
std::cout << " node map = node distribution over processors" << std::endl;
std::cout << argc << std::endl;
}
#endif //ifdef CurlCurlAndMassAreSeparate
#ifdef ML_MPI
MPI_Finalize();
#endif
exit(1);
}
Epetra_Map *edgeMap, *nodeMap;
Epetra_CrsMatrix *CCplusM=NULL, *CurlCurl=NULL, *Mass=NULL, *T=NULL, *Kn=NULL;
// ================================================= //
// READ IN MAPS FROM FILE //
// ================================================= //
// every processor reads this in
#ifdef CurlCurlAndMassAreSeparate
if (argc > 5)
#else
if (argc > 4)
#endif
{
datafile = argv[5];
if (Comm.MyPID() == 0) {
printf("Reading in edge map from %s ...\n",datafile);
fflush(stdout);
}
EpetraExt::MatrixMarketFileToMap(datafile, Comm, edgeMap);
datafile = argv[6];
if (Comm.MyPID() == 0) {
printf("Reading in node map from %s ...\n",datafile);
fflush(stdout);
}
EpetraExt::MatrixMarketFileToMap(datafile, Comm, nodeMap);
}
else { // linear maps
// Read the T matrix to determine the map sizes
// and then construct linear maps
if (Comm.MyPID() == 0)
printf("Using linear edge and node maps ...\n");
const int lineLength = 1025;
char line[lineLength];
FILE *handle;
int M,N,NZ;
#ifdef CurlCurlAndMassAreSeparate
handle = fopen(argv[3],"r");
#else
handle = fopen(argv[2],"r");
#endif
if (handle == 0) EPETRA_CHK_ERR(-1); // file not found
// Strip off header lines (which start with "%")
do {
if(fgets(line, lineLength, handle)==0) {if (handle!=0) fclose(handle);}
} while (line[0] == '%');
// Get problem dimensions: M, N, NZ
if(sscanf(line,"%d %d %d", &M, &N, &NZ)==0) {if (handle!=0) fclose(handle);}
fclose(handle);
edgeMap = new Epetra_Map(M,0,Comm);
nodeMap = new Epetra_Map(N,0,Comm);
}
// ===================================================== //
// READ IN MATRICES FROM FILE //
// ===================================================== //
#ifdef CurlCurlAndMassAreSeparate
for (int i = 1; i <5; i++) {
datafile = argv[i];
if (Comm.MyPID() == 0) {
printf("reading %s ....\n",datafile); fflush(stdout);
}
switch (i) {
case 1: //Curl
MatrixMarketFileToCrsMatrix(datafile,*edgeMap,*edgeMap,*edgeMap,CurlCurl);
break;
case 2: //Mass
MatrixMarketFileToCrsMatrix(datafile, *edgeMap, *edgeMap, *edgeMap, Mass);
break;
case 3: //Gradient
MatrixMarketFileToCrsMatrix(datafile, *edgeMap, *edgeMap,*nodeMap, T);
break;
case 4: //Auxiliary nodal matrix
MatrixMarketFileToCrsMatrix(datafile, *nodeMap,*nodeMap, *nodeMap, Kn);
break;
} //switch
} //for (int i = 1; i <5; i++)
#else
for (int i = 1; i <4; i++) {
datafile = argv[i];
if (Comm.MyPID() == 0) {
printf("reading %s ....\n",datafile); fflush(stdout);
}
switch (i) {
case 1: //Edge element matrix
MatrixMarketFileToCrsMatrix(datafile,*edgeMap,*edgeMap,*edgeMap,CCplusM);
break;
case 2: //Gradient
MatrixMarketFileToCrsMatrix(datafile, *edgeMap, *edgeMap, *nodeMap, T);
break;
case 3: //Auxiliary nodal matrix
MatrixMarketFileToCrsMatrix(datafile, *nodeMap, *nodeMap, *nodeMap, Kn);
break;
} //switch
} //for (int i = 1; i <4; i++)
#endif //ifdef CurlCurlAndMassAreSeparate
// ==================================================== //
// S E T U P O F M L P R E C O N D I T I O N E R //
// ==================================================== //
Teuchos::ParameterList MLList;
int *options = new int[AZ_OPTIONS_SIZE];
double *params = new double[AZ_PARAMS_SIZE];
ML_Epetra::SetDefaults("maxwell", MLList, options, params);
MLList.set("ML output", 10);
MLList.set("aggregation: type", "Uncoupled");
MLList.set("coarse: max size", 15);
MLList.set("aggregation: threshold", 0.0);
//MLList.set("negative conductivity",true);
//MLList.set("smoother: type", "Jacobi");
MLList.set("subsmoother: type", "symmetric Gauss-Seidel");
//MLList.set("max levels", 2);
// coarse level solve
MLList.set("coarse: type", "Amesos-KLU");
//MLList.set("coarse: type", "Hiptmair");
//MLList.set("coarse: type", "Jacobi");
//MLList.set("dump matrix: enable", true);
#ifdef CurlCurlAndMassAreSeparate
//Create the matrix of interest.
CCplusM = Epetra_MatrixAdd(CurlCurl,Mass,1.0);
#endif
#ifdef CurlCurlAndMassAreSeparate
ML_Epetra::MultiLevelPreconditioner * MLPrec =
new ML_Epetra::MultiLevelPreconditioner(*CurlCurl, *Mass, *T, *Kn, MLList);
// Comment out the line above and uncomment the next one if you have
// mass and curl separately but want to precondition as if they are added
// together.
/*
ML_Epetra::MultiLevelPreconditioner * MLPrec =
new ML_Epetra::MultiLevelPreconditioner(*CCplusM, *T, *Kn, MLList);
*/
#else
ML_Epetra::MultiLevelPreconditioner * MLPrec =
new ML_Epetra::MultiLevelPreconditioner(*CCplusM, *T, *Kn, MLList);
#endif //ifdef CurlCurlAndMassAreSeparate
MLPrec->PrintUnused(0);
// ========================================================= //
// D E F I N I T I O N O F A Z T E C O O P R O B L E M //
// ========================================================= //
// create left-hand side and right-hand side, and populate them with
// data from file. Both vectors are defined on the domain map of the
// edge matrix.
// Epetra_Vectors can be created in View mode, to accept pointers to
// double vectors.
if (Comm.MyPID() == 0)
std::cout << "Putting in a zero initial guess and random rhs (in the range of S+M)" << std::endl;
Epetra_Vector x(CCplusM->DomainMap());
x.Random();
Epetra_Vector rhs(CCplusM->DomainMap());
CCplusM->Multiply(false,x,rhs);
x.PutScalar(0.0);
double vecnorm;
rhs.Norm2(&vecnorm);
if (Comm.MyPID() == 0) std::cout << "||rhs|| = " << vecnorm << std::endl;
x.Norm2(&vecnorm);
if (Comm.MyPID() == 0) std::cout << "||x|| = " << vecnorm << std::endl;
// for AztecOO, we need an Epetra_LinearProblem
Epetra_LinearProblem Problem(CCplusM,&x,&rhs);
// AztecOO Linear problem
AztecOO solver(Problem);
// set MLPrec as precondititoning operator for AztecOO linear problem
//std::cout << "no ml preconditioner!!!" << std::endl;
solver.SetPrecOperator(MLPrec);
//solver.SetAztecOption(AZ_precond, AZ_Jacobi);
// a few options for AztecOO
solver.SetAztecOption(AZ_solver, AZ_cg);
solver.SetAztecOption(AZ_output, 1);
solver.Iterate(15, 1e-3);
// =============== //
// C L E A N U P //
// =============== //
delete MLPrec; // destroy phase prints out some information
delete CurlCurl;
delete CCplusM;
delete Mass;
delete T;
delete Kn;
delete nodeMap;
delete edgeMap;
delete [] params;
delete [] options;
ML_Comm_Destroy(&mlcomm);
} //avoids compiler error about jumping over initialization
droppedRankLabel:
#ifdef ML_MPI
MPI_Finalize();
#endif
return 0;
} //main
#else
#include <stdlib.h>
#include <stdio.h>
#ifdef HAVE_MPI
#include "mpi.h"
#endif
int main(int argc, char *argv[])
{
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
#endif
puts("Please configure ML with:");
#if !defined(HAVE_ML_EPETRA)
puts("--enable-epetra");
#endif
#if !defined(HAVE_ML_TEUCHOS)
puts("--enable-teuchos");
#endif
#if !defined(HAVE_ML_EPETRAEXT)
puts("--enable-epetraext");
#endif
#if !defined(HAVE_ML_AZTECOO)
puts("--enable-aztecoo");
#endif
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return 0;
}
int main(int argc, char *argv[])
{
int Nnodes=32*32; /* Total number of nodes in the problem.*/
/* 'Nnodes' must be a perfect square. */
struct user_partition Edge_Partition = {NULL, NULL,0,0,NULL,0,0,0},
Node_Partition = {NULL, NULL,0,0,NULL,0,0,0};
int proc_config[AZ_PROC_SIZE];
#ifdef ML_MPI
MPI_Init(&argc,&argv);
#endif
AZ_set_proc_config(proc_config, COMMUNICATOR);
ML_Comm* comm;
ML_Comm_Create(&comm);
Node_Partition.Nglobal = Nnodes;
Edge_Partition.Nglobal = Node_Partition.Nglobal*2;
user_partition_nodes(&Node_Partition);
user_partition_edges(&Edge_Partition, &Node_Partition);
AZ_MATRIX * AZ_Ke = user_Ke_build(&Edge_Partition);
AZ_MATRIX * AZ_Kn = user_Kn_build(&Node_Partition);
// convert (put wrappers) from Aztec matrices to ML_Operator's
ML_Operator * ML_Ke, * ML_Kn, * ML_Tmat;
ML_Ke = ML_Operator_Create( comm );
ML_Kn = ML_Operator_Create( comm );
AZ_convert_aztec_matrix_2ml_matrix(AZ_Ke,ML_Ke,proc_config);
AZ_convert_aztec_matrix_2ml_matrix(AZ_Kn,ML_Kn,proc_config);
ML_Tmat = user_T_build(&Edge_Partition, &Node_Partition,
ML_Kn, comm);
Epetra_CrsMatrix * Epetra_Kn, * Epetra_Ke, * Epetra_T;
int MaxNumNonzeros;
double CPUTime;
ML_Operator2EpetraCrsMatrix(ML_Ke,Epetra_Ke,
MaxNumNonzeros,
true,CPUTime);
ML_Operator2EpetraCrsMatrix(ML_Kn,
Epetra_Kn,MaxNumNonzeros,
true,CPUTime);
ML_Operator2EpetraCrsMatrix(ML_Tmat,Epetra_T,MaxNumNonzeros,
true,CPUTime);
Teuchos::ParameterList MLList;
ML_Epetra::SetDefaults("maxwell", MLList);
MLList.set("ML output", 0);
MLList.set("aggregation: type", "Uncoupled");
MLList.set("coarse: max size", 30);
MLList.set("aggregation: threshold", 0.0);
MLList.set("coarse: type", "Amesos-KLU");
ML_Epetra::MultiLevelPreconditioner * MLPrec =
new ML_Epetra::MultiLevelPreconditioner(*Epetra_Ke, *Epetra_T, *Epetra_Kn,
MLList);
Epetra_Vector LHS(Epetra_Ke->DomainMap()); LHS.Random();
Epetra_Vector RHS(Epetra_Ke->DomainMap()); RHS.PutScalar(1.0);
Epetra_LinearProblem Problem(Epetra_Ke,&LHS,&RHS);
AztecOO solver(Problem);
solver.SetPrecOperator(MLPrec);
solver.SetAztecOption(AZ_solver, AZ_cg_condnum);
solver.SetAztecOption(AZ_output, 32);
solver.Iterate(500, 1e-8);
// ========================= //
// compute the real residual //
// ========================= //
Epetra_Vector RHScomp(Epetra_Ke->DomainMap());
int ierr;
ierr = Epetra_Ke->Multiply(false, LHS, RHScomp);
assert(ierr==0);
Epetra_Vector resid(Epetra_Ke->DomainMap());
ierr = resid.Update(1.0, RHS, -1.0, RHScomp, 0.0);
assert(ierr==0);
double residual;
ierr = resid.Norm2(&residual);
assert(ierr==0);
if (proc_config[AZ_node] == 0) {
std::cout << std::endl;
std::cout << "==> Residual = " << residual << std::endl;
std::cout << std::endl;
}
// =============== //
// C L E A N U P //
// =============== //
delete MLPrec; // destroy phase prints out some information
delete Epetra_Kn;
delete Epetra_Ke;
delete Epetra_T;
ML_Operator_Destroy( &ML_Ke );
ML_Operator_Destroy( &ML_Kn );
ML_Comm_Destroy( &comm );
if (Edge_Partition.my_local_ids != NULL) free(Edge_Partition.my_local_ids);
if (Node_Partition.my_local_ids != NULL) free(Node_Partition.my_local_ids);
if (Node_Partition.my_global_ids != NULL) free(Node_Partition.my_global_ids);
if (Edge_Partition.my_global_ids != NULL) free(Edge_Partition.my_global_ids);
if (Node_Partition.needed_external_ids != NULL)
free(Node_Partition.needed_external_ids);
if (Edge_Partition.needed_external_ids != NULL)
free(Edge_Partition.needed_external_ids);
if (AZ_Ke!= NULL) {
AZ_free(AZ_Ke->bindx);
AZ_free(AZ_Ke->val);
AZ_free(AZ_Ke->data_org);
AZ_matrix_destroy(&AZ_Ke);
}
if (AZ_Kn!= NULL) {
AZ_free(AZ_Kn->bindx);
AZ_free(AZ_Kn->val);
AZ_free(AZ_Kn->data_org);
AZ_matrix_destroy(&AZ_Kn);
}
ML_Operator_Destroy(&ML_Tmat);
if (residual > 1e-5) {
std::cout << "`MultiLevelPreconditioner_Maxwell.exe' failed!" << std::endl;
exit(EXIT_FAILURE);
}
#ifdef ML_MPI
MPI_Finalize();
#endif
if (proc_config[AZ_node] == 0)
std::cout << "`MultiLevelPreconditioner_Maxwell.exe' passed!" << std::endl;
exit(EXIT_SUCCESS);
}
