Example: mpiexec -n <np> ./ex130 -f <matrix binary file> -mat_solver_package 1 -mat_superlu_equil \n\n";
#include <petscmat.h>
int main(int argc,char **args)
{
Mat A,F;
Vec u,x,b;
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInt m,n,nfact,ipack=0;
PetscReal norm,tol=1.e-12,Anorm;
IS perm,iperm;
MatFactorInfo info;
PetscBool flg,testMatSolve=PETSC_TRUE;
PetscViewer fd; /* viewer */
char file[PETSC_MAX_PATH_LEN]; /* input file name */
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
/* Determine file from which we read the matrix A */
ierr = PetscOptionsGetString(NULL,NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,1,"Must indicate binary file with the -f option");
/* Load matrix A */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecLoad(b,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
if (m != n) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ, "This example is not intended for rectangular matrices (%d, %d)", m, n);
ierr = MatNorm(A,NORM_INFINITY,&Anorm);CHKERRQ(ierr);
/* Create vectors */
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);CHKERRQ(ierr); /* save the true solution */
/* Test LU Factorization */
ierr = MatGetOrdering(A,MATORDERINGNATURAL,&perm,&iperm);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_solver_package",&ipack,NULL);CHKERRQ(ierr);
switch (ipack) {
case 1:
#if defined(PETSC_HAVE_SUPERLU)
if (!rank) printf(" SUPERLU LU:\n");
ierr = MatGetFactor(A,MATSOLVERSUPERLU,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
break;
#endif
case 2:
#if defined(PETSC_HAVE_MUMPS)
if (!rank) printf(" MUMPS LU:\n");
ierr = MatGetFactor(A,MATSOLVERMUMPS,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
{
/* test mumps options */
PetscInt icntl_7 = 5;
ierr = MatMumpsSetIcntl(F,7,icntl_7);CHKERRQ(ierr);
}
break;
#endif
default:
if (!rank) printf(" PETSC LU:\n");
ierr = MatGetFactor(A,MATSOLVERPETSC,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
}
info.fill = 5.0;
ierr = MatLUFactorSymbolic(F,A,perm,iperm,&info);CHKERRQ(ierr);
for (nfact = 0; nfact < 1; nfact++) {
if (!rank) printf(" %d-the LU numfactorization \n",nfact);
ierr = MatLUFactorNumeric(F,A,&info);CHKERRQ(ierr);
/* Test MatSolve() */
if (testMatSolve) {
ierr = MatSolve(F,b,x);CHKERRQ(ierr);
/* Check the residual */
ierr = MatMult(A,x,u);CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(u,NORM_INFINITY,&norm);CHKERRQ(ierr);
if (norm > tol) {
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatSolve: rel residual %g/%g = %g, LU numfact %d\n",norm,Anorm,norm/Anorm,nfact);CHKERRQ(ierr);
}
}
}
}
/* Free data structures */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
ierr = ISDestroy(&perm);CHKERRQ(ierr);
ierr = ISDestroy(&iperm);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
// =====================================================
void PetscPreconditioner::set_petsc_preconditioner_type
(const PreconditionerType & preconditioner_type, PC & pc, const int ¶llelOverlapping) {
int ierr = 0;
switch(preconditioner_type) {
case IDENTITY_PRECOND:
ierr = PCSetType(pc, (char*) PCNONE);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case CHOLESKY_PRECOND:
ierr = PCSetType(pc, (char*) PCCHOLESKY);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case ICC_PRECOND:
ierr = PCSetType(pc, (char*) PCICC);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case ILU_PRECOND:
{
int nprocs;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs); //TODO
// In serial, just set the ILU preconditioner type
if(nprocs == 1)
{
ierr = PCSetType(pc, (char*) PCILU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
}
else
{
// But PETSc has no truly parallel ILU, instead you have to set
// an actual parallel preconditioner (e.g. block Jacobi (parlleloverlapping 0) or ASM (parlleloverlapping >0))
// and then assign ILU sub-preconditioners.
set_petsc_preconditioner_type(ASM_PRECOND, pc);
PCASMSetOverlap(pc, parallelOverlapping);
PCSetUp(pc);
// Set ILU as the sub preconditioner type
set_petsc_subpreconditioner_type(PCILU, pc);
}
break;
}
case LU_PRECOND: {
int nprocs;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if(nprocs == 1) {
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
}
else {
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERMUMPS);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetUpMatSolverPackage(pc);
CHKERRABORT(MPI_COMM_WORLD, ierr);
Mat F;
ierr = PCFactorGetMatrix(pc, &F);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = MatMumpsSetIcntl(F, 14, 30);
CHKERRABORT(MPI_COMM_WORLD, ierr);
}
break;
}
case SLU_PRECOND:
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERSUPERLU_DIST);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break; //here we set the SuperLU_dist solver package
case MLU_PRECOND: //here we set the MUMPS parallel direct solver package
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERMUMPS);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetUpMatSolverPackage(pc);
CHKERRABORT(MPI_COMM_WORLD, ierr);
Mat F;
ierr = PCFactorGetMatrix(pc, &F);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = MatMumpsSetIcntl(F, 14, 30);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case ULU_PRECOND: //here we set the Umfpack serial direct solver package
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERUMFPACK);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetUpMatSolverPackage(pc);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case MCC_PRECOND:
ierr = PCSetType(pc, (char*) PCCHOLESKY);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERMUMPS);
CHKERRABORT(MPI_COMM_WORLD, ierr); //here we set the MUMPS parallel direct solver package
break;
case ASM_PRECOND:
ierr = PCSetType(pc, (char*) PCASM);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case FIELDSPLIT_PRECOND:
ierr = PCSetType(pc, (char*) PCFIELDSPLIT);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case JACOBI_PRECOND:
ierr = PCSetType(pc, (char*) PCJACOBI);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case BLOCK_JACOBI_PRECOND:
ierr = PCSetType(pc, (char*) PCBJACOBI);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case SOR_PRECOND:
ierr = PCSetType(pc, (char*) PCSOR);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case EISENSTAT_PRECOND:
ierr = PCSetType(pc, (char*) PCEISENSTAT);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case AMG_PRECOND:
ierr = PCSetType(pc, (char*) PCHYPRE);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case MG_PRECOND:
ierr = PCSetType(pc, (char*) PCMG);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case LSC_PRECOND:
ierr = PCSetType(pc, (char*) PCLSC);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
#if !(PETSC_VERSION_LESS_THAN(2,1,2))
// Only available for PETSC >= 2.1.2
case USER_PRECOND:
ierr = PCSetType(pc, (char*) PCMAT);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
#endif
case SHELL_PRECOND:
ierr = PCSetType(pc, (char*) PCSHELL);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
default:
std::cerr << "ERROR: Unsupported PETSC Preconditioner: "
<< preconditioner_type << std::endl
<< "Continuing with PETSC defaults" << std::endl;
}
//Let the commandline override stuff
if(preconditioner_type != AMG_PRECOND && preconditioner_type != MG_PRECOND) PCSetFromOptions(pc); //!!!!!!
}
void PETSC_STDCALL matmumpsseticntl_(Mat F,PetscInt *idx,PetscInt *icntl, int *__ierr ){
*__ierr = MatMumpsSetIcntl(
(Mat)PetscToPointer((F) ),*idx,*icntl);
}
int main(int argc,char **args)
{
Mat A,RHS,C,F,X;
Vec u,x,b;
PetscErrorCode ierr;
PetscMPIInt rank,nproc;
PetscInt i,m,n,nfact,nsolve,nrhs,ipack=0;
PetscScalar *array,rval;
PetscReal norm,tol=1.e-12;
IS perm,iperm;
MatFactorInfo info;
PetscRandom rand;
PetscBool flg,testMatSolve=PETSC_TRUE,testMatMatSolve=PETSC_TRUE;
PetscViewer fd; /* viewer */
char file[PETSC_MAX_PATH_LEN]; /* input file name */
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &nproc);CHKERRQ(ierr);
/* Determine file from which we read the matrix A */
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,1,"Must indicate binary file with the -f option");
/* Load matrix A */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
if (m != n) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ, "This example is not intended for rectangular matrices (%d, %d)", m, n);
/* Create dense matrix C and X; C holds true solution with identical colums */
nrhs = 2;
ierr = PetscOptionsGetInt(NULL,"-nrhs",&nrhs,NULL);CHKERRQ(ierr);
if (!rank) printf("ex125: nrhs %d\n",nrhs);
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,m,PETSC_DECIDE,PETSC_DECIDE,nrhs);CHKERRQ(ierr);
ierr = MatSetType(C,MATDENSE);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatSetRandom(C,rand);CHKERRQ(ierr);
ierr = MatDuplicate(C,MAT_DO_NOT_COPY_VALUES,&X);CHKERRQ(ierr);
/* Create vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);CHKERRQ(ierr); /* save the true solution */
/* Test LU Factorization */
ierr = MatGetOrdering(A,MATORDERINGND,&perm,&iperm);CHKERRQ(ierr);
/*ierr = ISView(perm,PETSC_VIEWER_STDOUT_WORLD);*/
/*ierr = ISView(perm,PETSC_VIEWER_STDOUT_SELF);*/
ierr = PetscOptionsGetInt(NULL,"-mat_solver_package",&ipack,NULL);CHKERRQ(ierr);
switch (ipack) {
case 0:
#if defined(PETSC_HAVE_SUPERLU)
if (!rank) printf(" SUPERLU LU:\n");
ierr = MatGetFactor(A,MATSOLVERSUPERLU,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
break;
#endif
case 1:
#if defined(PETSC_HAVE_SUPERLU_DIST)
if (!rank) printf(" SUPERLU_DIST LU:\n");
ierr = MatGetFactor(A,MATSOLVERSUPERLU_DIST,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
break;
#endif
case 2:
#if defined(PETSC_HAVE_MUMPS)
if (!rank) printf(" MUMPS LU:\n");
ierr = MatGetFactor(A,MATSOLVERMUMPS,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
{
/* test mumps options */
PetscInt icntl_7 = 5;
ierr = MatMumpsSetIcntl(F,7,icntl_7);CHKERRQ(ierr);
}
break;
#endif
default:
if (!rank) printf(" PETSC LU:\n");
ierr = MatGetFactor(A,MATSOLVERPETSC,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
}
info.fill = 5.0;
ierr = MatLUFactorSymbolic(F,A,perm,iperm,&info);CHKERRQ(ierr);
for (nfact = 0; nfact < 2; nfact++) {
if (!rank) printf(" %d-the LU numfactorization \n",nfact);
ierr = MatLUFactorNumeric(F,A,&info);CHKERRQ(ierr);
/* Test MatMatSolve() */
/*
if ((ipack == 0 || ipack == 2) && testMatMatSolve) {
printf(" MatMatSolve() is not implemented for this package. Skip the testing.\n");
testMatMatSolve = PETSC_FALSE;
}
*/
if (testMatMatSolve) {
if (!nfact) {
ierr = MatMatMult(A,C,MAT_INITIAL_MATRIX,2.0,&RHS);CHKERRQ(ierr);
} else {
ierr = MatMatMult(A,C,MAT_REUSE_MATRIX,2.0,&RHS);CHKERRQ(ierr);
}
for (nsolve = 0; nsolve < 2; nsolve++) {
if (!rank) printf(" %d-the MatMatSolve \n",nsolve);
ierr = MatMatSolve(F,RHS,X);CHKERRQ(ierr);
/* Check the error */
ierr = MatAXPY(X,-1.0,C,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatNorm(X,NORM_FROBENIUS,&norm);CHKERRQ(ierr);
if (norm > tol) {
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"1st MatMatSolve: Norm of error %g, nsolve %d\n",norm,nsolve);CHKERRQ(ierr);
}
}
}
}
/* Test MatSolve() */
if (testMatSolve) {
for (nsolve = 0; nsolve < 2; nsolve++) {
ierr = VecGetArray(x,&array);CHKERRQ(ierr);
for (i=0; i<m; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
array[i] = rval;
}
ierr = VecRestoreArray(x,&array);CHKERRQ(ierr);
ierr = VecCopy(x,u);CHKERRQ(ierr);
ierr = MatMult(A,x,b);CHKERRQ(ierr);
if (!rank) printf(" %d-the MatSolve \n",nsolve);
ierr = MatSolve(F,b,x);CHKERRQ(ierr);
/* Check the error */
ierr = VecAXPY(u,-1.0,x);CHKERRQ(ierr); /* u <- (-1.0)x + u */
ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = MatMult(A,x,u);CHKERRQ(ierr); /* u = A*x */
PetscReal resi;
ierr = VecAXPY(u,-1.0,b);CHKERRQ(ierr); /* u <- (-1.0)b + u */
ierr = VecNorm(u,NORM_2,&resi);CHKERRQ(ierr);
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatSolve: Norm of error %g, resi %g, LU numfact %d\n",norm,resi,nfact);CHKERRQ(ierr);
}
}
}
}
}
/* Free data structures */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
ierr = MatDestroy(&X);CHKERRQ(ierr);
if (testMatMatSolve) {
ierr = MatDestroy(&RHS);CHKERRQ(ierr);
}
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = ISDestroy(&perm);CHKERRQ(ierr);
ierr = ISDestroy(&iperm);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
