PetscErrorCode MatDestroy_SuperLU_DIST(Mat A)
{
PetscErrorCode ierr;
Mat_SuperLU_DIST *lu = (Mat_SuperLU_DIST*)A->spptr;
PetscBool flg;
PetscFunctionBegin;
if (lu && lu->CleanUpSuperLU_Dist) {
/* Deallocate SuperLU_DIST storage */
if (lu->MatInputMode == GLOBAL) {
Destroy_CompCol_Matrix_dist(&lu->A_sup);
} else {
Destroy_CompRowLoc_Matrix_dist(&lu->A_sup);
if ( lu->options.SolveInitialized ) {
#if defined(PETSC_USE_COMPLEX)
zSolveFinalize(&lu->options, &lu->SOLVEstruct);
#else
dSolveFinalize(&lu->options, &lu->SOLVEstruct);
#endif
}
}
Destroy_LU(A->cmap->N, &lu->grid, &lu->LUstruct);
ScalePermstructFree(&lu->ScalePermstruct);
LUstructFree(&lu->LUstruct);
/* Release the SuperLU_DIST process grid. */
superlu_gridexit(&lu->grid);
ierr = MPI_Comm_free(&(lu->comm_superlu));CHKERRQ(ierr);
}
ierr = PetscFree(A->spptr);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatDestroy_SeqAIJ(A);CHKERRQ(ierr);
} else {
ierr = MatDestroy_MPIAIJ(A);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
superlu_options_t options;
SuperLUStat_t stat;
SuperMatrix A;
NRformat_loc *Astore;
ScalePermstruct_t ScalePermstruct;
LUstruct_t LUstruct;
SOLVEstruct_t SOLVEstruct;
gridinfo_t grid;
double *berr;
double *b, *b1, *xtrue, *nzval, *nzval1;
int_t *colind, *colind1, *rowptr, *rowptr1;
int_t i, j, m, n, nnz_loc, m_loc, fst_row;
int nprow, npcol;
int iam, info, ldb, ldx, nrhs;
char **cpp, c;
FILE *fp, *fopen();
nprow = 1; /* Default process rows. */
npcol = 1; /* Default process columns. */
nrhs = 1; /* Number of right-hand side. */
/* ------------------------------------------------------------
INITIALIZE MPI ENVIRONMENT.
------------------------------------------------------------*/
MPI_Init( &argc, &argv );
/* Parse command line argv[]. */
for (cpp = argv+1; *cpp; ++cpp) {
if ( **cpp == '-' ) {
c = *(*cpp+1);
++cpp;
switch (c) {
case 'h':
printf("Options:\n");
printf("\t-r <int>: process rows (default %d)\n", nprow);
printf("\t-c <int>: process columns (default %d)\n", npcol);
exit(0);
break;
case 'r': nprow = atoi(*cpp);
break;
case 'c': npcol = atoi(*cpp);
break;
}
} else { /* Last arg is considered a filename */
if ( !(fp = fopen(*cpp, "r")) ) {
ABORT("File does not exist");
}
break;
}
}
/* ------------------------------------------------------------
INITIALIZE THE SUPERLU PROCESS GRID.
------------------------------------------------------------*/
superlu_gridinit(MPI_COMM_WORLD, nprow, npcol, &grid);
/* Bail out if I do not belong in the grid. */
iam = grid.iam;
if ( iam >= nprow * npcol ) goto out;
if ( !iam ) {
printf("Input matrix file: %s\n", *cpp);
printf("\tProcess grid\t%d X %d\n", grid.nprow, grid.npcol);
}
#if ( DEBUGlevel>=1 )
CHECK_MALLOC(iam, "Enter main()");
#endif
/* ------------------------------------------------------------
GET THE MATRIX FROM FILE AND SETUP THE RIGHT HAND SIDE.
------------------------------------------------------------*/
dcreate_matrix(&A, nrhs, &b, &ldb, &xtrue, &ldx, fp, &grid);
if ( !(b1 = doubleMalloc_dist(ldb * nrhs)) )
ABORT("Malloc fails for b1[]");
for (j = 0; j < nrhs; ++j)
for (i = 0; i < ldb; ++i) b1[i+j*ldb] = b[i+j*ldb];
if ( !(berr = doubleMalloc_dist(nrhs)) )
ABORT("Malloc fails for berr[].");
m = A.nrow;
n = A.ncol;
/* Save a copy of the matrix A. */
Astore = (NRformat_loc *) A.Store;
nnz_loc = Astore->nnz_loc;
m_loc = Astore->m_loc;
fst_row = Astore->fst_row;
nzval = Astore->nzval;
colind = Astore->colind;
rowptr = Astore->rowptr;
nzval1 = doubleMalloc_dist(nnz_loc);
colind1 = intMalloc_dist(nnz_loc);
rowptr1 = intMalloc_dist(m_loc+1);
for (i = 0; i < nnz_loc; ++i) {
nzval1[i] = nzval[i];
colind1[i] = colind[i];
}
for (i = 0; i < m_loc+1; ++i) rowptr1[i] = rowptr[i];
/* ------------------------------------------------------------
WE SOLVE THE LINEAR SYSTEM FOR THE FIRST TIME.
------------------------------------------------------------*/
/* Set the default input options:
options.Fact = DOFACT;
options.Equil = YES;
options.ColPerm = METIS_AT_PLUS_A;
options.RowPerm = LargeDiag;
options.ReplaceTinyPivot = YES;
options.Trans = NOTRANS;
options.IterRefine = DOUBLE;
options.SolveInitialized = NO;
options.RefineInitialized = NO;
options.PrintStat = YES;
*/
set_default_options_dist(&options);
if (!iam) {
print_sp_ienv_dist(&options);
print_options_dist(&options);
}
/* Initialize ScalePermstruct and LUstruct. */
ScalePermstructInit(m, n, &ScalePermstruct);
LUstructInit(n, &LUstruct);
/* Initialize the statistics variables. */
PStatInit(&stat);
/* Call the linear equation solver: factorize and solve. */
pdgssvx(&options, &A, &ScalePermstruct, b, ldb, nrhs, &grid,
&LUstruct, &SOLVEstruct, berr, &stat, &info);
/* Check the accuracy of the solution. */
pdinf_norm_error(iam, m_loc, nrhs, b, ldb, xtrue, ldx, &grid);
PStatPrint(&options, &stat, &grid); /* Print the statistics. */
PStatFree(&stat);
Destroy_CompRowLoc_Matrix_dist(&A); /* Deallocate storage of matrix A. */
SUPERLU_FREE(b); /* Free storage of right-hand side. */
/* ------------------------------------------------------------
NOW WE SOLVE ANOTHER LINEAR SYSTEM.
THE MATRIX A HAS THE SAME SPARSITY PATTERN AND THE SIMILAR
NUMERICAL VALUES AS THAT IN A PREVIOUS SYSTEM.
------------------------------------------------------------*/
options.Fact = SamePattern_SameRowPerm;
PStatInit(&stat); /* Initialize the statistics variables. */
/* Set up the local A in NR_loc format */
dCreate_CompRowLoc_Matrix_dist(&A, m, n, nnz_loc, m_loc, fst_row,
nzval1, colind1, rowptr1,
SLU_NR_loc, SLU_D, SLU_GE);
/* Solve the linear system. */
pdgssvx(&options, &A, &ScalePermstruct, b1, ldb, nrhs, &grid,
&LUstruct, &SOLVEstruct, berr, &stat, &info);
/* Check the accuracy of the solution. */
if ( !iam )
printf("Solve a system with the same pattern and similar values.\n");
pdinf_norm_error(iam, m_loc, nrhs, b1, ldb, xtrue, ldx, &grid);
/* Print the statistics. */
PStatPrint(&options, &stat, &grid);
/* ------------------------------------------------------------
DEALLOCATE STORAGE.
------------------------------------------------------------*/
PStatFree(&stat);
Destroy_CompRowLoc_Matrix_dist(&A); /* Deallocate storage of matrix A. */
Destroy_LU(n, &grid, &LUstruct); /* Deallocate storage associated with
the L and U matrices. */
ScalePermstructFree(&ScalePermstruct);
LUstructFree(&LUstruct); /* Deallocate the structure of L and U.*/
if ( options.SolveInitialized ) {
dSolveFinalize(&options, &SOLVEstruct);
}
SUPERLU_FREE(b1); /* Free storage of right-hand side. */
SUPERLU_FREE(xtrue); /* Free storage of the exact solution. */
SUPERLU_FREE(berr);
/* ------------------------------------------------------------
RELEASE THE SUPERLU PROCESS GRID.
------------------------------------------------------------*/
out:
superlu_gridexit(&grid);
/* ------------------------------------------------------------
TERMINATES THE MPI EXECUTION ENVIRONMENT.
------------------------------------------------------------*/
MPI_Finalize();
#if ( DEBUGlevel>=1 )
CHECK_MALLOC(iam, "Exit main()");
#endif
}
void f_dSolveFinalize(fptr *options, fptr *SOLVEstruct)
{
dSolveFinalize((superlu_options_t *) *options,
(SOLVEstruct_t *) *SOLVEstruct);
}
int main(int argc, char *argv[])
{
superlu_dist_options_t options;
SuperLUStat_t stat;
SuperMatrix A;
ScalePermstruct_t ScalePermstruct;
LUstruct_t LUstruct;
SOLVEstruct_t SOLVEstruct;
gridinfo_t grid1, grid2;
double *berr;
double *a, *b, *xtrue;
int_t *asub, *xa;
int_t i, j, m, n;
int nprow, npcol, ldumap, p;
int_t usermap[6];
int iam, info, ldb, ldx, nprocs;
int nrhs = 1; /* Number of right-hand side. */
char **cpp, c;
FILE *fp, *fopen();
int cpp_defs();
/* ------------------------------------------------------------
INITIALIZE MPI ENVIRONMENT.
------------------------------------------------------------*/
MPI_Init( &argc, &argv );
MPI_Comm_size( MPI_COMM_WORLD, &nprocs );
if ( nprocs < 10 ) {
fprintf(stderr, "Requires at least 10 processes\n");
exit(-1);
}
/* Parse command line argv[]. */
for (cpp = argv+1; *cpp; ++cpp) {
if ( **cpp == '-' ) {
c = *(*cpp+1);
++cpp;
switch (c) {
case 'h':
printf("Options:\n");
printf("\t-r <int>: process rows (default %d)\n", nprow);
printf("\t-c <int>: process columns (default %d)\n", npcol);
exit(0);
break;
case 'r': nprow = atoi(*cpp);
break;
case 'c': npcol = atoi(*cpp);
break;
}
} else { /* Last arg is considered a filename */
if ( !(fp = fopen(*cpp, "r")) ) {
ABORT("File does not exist");
}
break;
}
}
/* ------------------------------------------------------------
INITIALIZE THE SUPERLU PROCESS GRID 1.
------------------------------------------------------------*/
nprow = 2;
npcol = 3;
ldumap = 2;
p = 0; /* Grid 1 starts from process 0. */
for (i = 0; i < nprow; ++i)
for (j = 0; j < npcol; ++j) usermap[i+j*ldumap] = p++;
superlu_gridmap(MPI_COMM_WORLD, nprow, npcol, usermap, ldumap, &grid1);
/* ------------------------------------------------------------
INITIALIZE THE SUPERLU PROCESS GRID 2.
------------------------------------------------------------*/
nprow = 2;
npcol = 2;
ldumap = 2;
p = 6; /* Grid 2 starts from process 6. */
for (i = 0; i < nprow; ++i)
for (j = 0; j < npcol; ++j) usermap[i+j*ldumap] = p++;
superlu_gridmap(MPI_COMM_WORLD, nprow, npcol, usermap, ldumap, &grid2);
/* Bail out if I do not belong in any of the 2 grids. */
MPI_Comm_rank( MPI_COMM_WORLD, &iam );
if ( iam >= 10 ) goto out;
#if ( DEBUGlevel>=1 )
CHECK_MALLOC(iam, "Enter main()");
#endif
if ( iam >= 0 && iam < 6 ) { /* I am in grid 1. */
iam = grid1.iam; /* Get the logical number in the new grid. */
/* ------------------------------------------------------------
GET THE MATRIX FROM FILE AND SETUP THE RIGHT HAND SIDE.
------------------------------------------------------------*/
dcreate_matrix(&A, nrhs, &b, &ldb, &xtrue, &ldx, fp, &grid1);
if ( !(berr = doubleMalloc_dist(nrhs)) )
ABORT("Malloc fails for berr[].");
/* ------------------------------------------------------------
NOW WE SOLVE THE LINEAR SYSTEM.
------------------------------------------------------------*/
/* Set the default input options:
options.Fact = DOFACT;
options.Equil = YES;
options.ColPerm = METIS_AT_PLUS_A;
options.RowPerm = LargeDiag;
options.ReplaceTinyPivot = NO;
options.Trans = NOTRANS;
options.IterRefine = DOUBLE;
options.SolveInitialized = NO;
options.RefineInitialized = NO;
options.PrintStat = YES;
*/
set_default_options_dist(&options);
if (!iam) {
print_sp_ienv_dist(&options);
print_options_dist(&options);
}
m = A.nrow;
n = A.ncol;
/* Initialize ScalePermstruct and LUstruct. */
ScalePermstructInit(m, n, &ScalePermstruct);
LUstructInit(n, &LUstruct);
/* Initialize the statistics variables. */
PStatInit(&stat);
/* Call the linear equation solver. */
pdgssvx(&options, &A, &ScalePermstruct, b, ldb, nrhs, &grid1,
&LUstruct, &SOLVEstruct, berr, &stat, &info);
/* Check the accuracy of the solution. */
pdinf_norm_error(iam, ((NRformat_loc *)A.Store)->m_loc,
nrhs, b, ldb, xtrue, ldx, &grid1);
/* Print the statistics. */
PStatPrint(&options, &stat, &grid1);
/* ------------------------------------------------------------
DEALLOCATE STORAGE.
------------------------------------------------------------*/
PStatFree(&stat);
Destroy_CompRowLoc_Matrix_dist(&A);
ScalePermstructFree(&ScalePermstruct);
Destroy_LU(n, &grid1, &LUstruct);
LUstructFree(&LUstruct);
if ( options.SolveInitialized ) {
dSolveFinalize(&options, &SOLVEstruct);
}
SUPERLU_FREE(b);
SUPERLU_FREE(xtrue);
SUPERLU_FREE(berr);
} else { /* I am in grid 2. */
iam = grid2.iam; /* Get the logical number in the new grid. */
/* ------------------------------------------------------------
GET THE MATRIX FROM FILE AND SETUP THE RIGHT HAND SIDE.
------------------------------------------------------------*/
dcreate_matrix(&A, nrhs, &b, &ldb, &xtrue, &ldx, fp, &grid2);
if ( !(berr = doubleMalloc_dist(nrhs)) )
ABORT("Malloc fails for berr[].");
/* ------------------------------------------------------------
NOW WE SOLVE THE LINEAR SYSTEM.
------------------------------------------------------------*/
/* Set the default input options:
options.Fact = DOFACT;
options.Equil = YES;
options.ColPerm = MMD_AT_PLUS_A;
options.RowPerm = LargeDiag;
options.ReplaceTinyPivot = YES;
options.Trans = NOTRANS;
options.IterRefine = DOUBLE;
options.SolveInitialized = NO;
options.RefineInitialized = NO;
options.PrintStat = YES;
*/
set_default_options_dist(&options);
m = A.nrow;
n = A.ncol;
/* Initialize ScalePermstruct and LUstruct. */
ScalePermstructInit(m, n, &ScalePermstruct);
LUstructInit(n, &LUstruct);
/* Initialize the statistics variables. */
PStatInit(&stat);
/* Call the linear equation solver. */
pdgssvx(&options, &A, &ScalePermstruct, b, ldb, nrhs, &grid2,
&LUstruct, &SOLVEstruct, berr, &stat, &info);
/* Check the accuracy of the solution. */
pdinf_norm_error(iam, ((NRformat_loc *)A.Store)->m_loc,
nrhs, b, ldb, xtrue, ldx, &grid2);
/* Print the statistics. */
PStatPrint(&options, &stat, &grid2);
/* ------------------------------------------------------------
DEALLOCATE STORAGE.
------------------------------------------------------------*/
PStatFree(&stat);
Destroy_CompRowLoc_Matrix_dist(&A);
ScalePermstructFree(&ScalePermstruct);
Destroy_LU(n, &grid2, &LUstruct);
LUstructFree(&LUstruct);
if ( options.SolveInitialized ) {
dSolveFinalize(&options, &SOLVEstruct);
}
SUPERLU_FREE(b);
SUPERLU_FREE(xtrue);
SUPERLU_FREE(berr);
}
/* ------------------------------------------------------------
RELEASE THE SUPERLU PROCESS GRIDS.
------------------------------------------------------------*/
superlu_gridexit(&grid1);
superlu_gridexit(&grid2);
out:
/* ------------------------------------------------------------
TERMINATES THE MPI EXECUTION ENVIRONMENT.
------------------------------------------------------------*/
MPI_Finalize();
#if ( DEBUGlevel>=1 )
CHECK_MALLOC(iam, "Exit main()");
#endif
}
PetscErrorCode MatMatSolve_SuperLU_DIST(Mat A,Mat B_mpi,Mat X)
{
Mat_SuperLU_DIST *lu = (Mat_SuperLU_DIST*)A->spptr;
PetscErrorCode ierr;
PetscMPIInt size;
PetscInt M=A->rmap->N,m=A->rmap->n,nrhs;
SuperLUStat_t stat;
double berr[1];
PetscScalar *bptr;
int info; /* SuperLU_Dist info code is ALWAYS an int, even with long long indices */
PetscBool flg;
PetscFunctionBegin;
if (lu->options.Fact != FACTORED) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"SuperLU_DIST options.Fact mush equal FACTORED");
ierr = PetscObjectTypeCompareAny((PetscObject)B_mpi,&flg,MATSEQDENSE,MATMPIDENSE,NULL);CHKERRQ(ierr);
if (!flg) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONG,"Matrix B must be MATDENSE matrix");
ierr = PetscObjectTypeCompareAny((PetscObject)X,&flg,MATSEQDENSE,MATMPIDENSE,NULL);CHKERRQ(ierr);
if (!flg) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONG,"Matrix X must be MATDENSE matrix");
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&size);CHKERRQ(ierr);
if (size > 1 && lu->MatInputMode == GLOBAL) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"MatInputMode=GLOBAL for nproc %d>1 is not supported",size);
/* size==1 or distributed mat input */
if (lu->options.SolveInitialized && !lu->matmatsolve_iscalled) {
/* communication pattern of SOLVEstruct is unlikely created for matmatsolve,
thus destroy it and create a new SOLVEstruct.
Otherwise it may result in memory corruption or incorrect solution
See src/mat/examples/tests/ex125.c */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zSolveFinalize",zSolveFinalize(&lu->options, &lu->SOLVEstruct));
#else
PetscStackCall("SuperLU_DIST:dSolveFinalize",dSolveFinalize(&lu->options, &lu->SOLVEstruct));
#endif
lu->options.SolveInitialized = NO;
}
ierr = MatCopy(B_mpi,X,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatGetSize(B_mpi,NULL,&nrhs);CHKERRQ(ierr);
PetscStackCall("SuperLU_DIST:PStatInit",PStatInit(&stat)); /* Initialize the statistics variables. */
ierr = MatDenseGetArray(X,&bptr);CHKERRQ(ierr);
if (lu->MatInputMode == GLOBAL) { /* size == 1 */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx_ABglobal",pzgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct,(doublecomplex*)bptr, M, nrhs,&lu->grid, &lu->LUstruct, berr, &stat, &info));
#else
PetscStackCall("SuperLU_DIST:pdgssvx_ABglobal",pdgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct,bptr, M, nrhs, &lu->grid, &lu->LUstruct, berr, &stat, &info));
#endif
} else { /* distributed mat input */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx",pzgssvx(&lu->options,&lu->A_sup,&lu->ScalePermstruct,(doublecomplex*)bptr,m,nrhs,&lu->grid, &lu->LUstruct,&lu->SOLVEstruct,berr,&stat,&info));
#else
PetscStackCall("SuperLU_DIST:pdgssvx",pdgssvx(&lu->options,&lu->A_sup,&lu->ScalePermstruct,bptr,m,nrhs,&lu->grid,&lu->LUstruct,&lu->SOLVEstruct,berr,&stat,&info));
#endif
}
if (info) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pdgssvx fails, info: %d\n",info);
ierr = MatDenseRestoreArray(X,&bptr);CHKERRQ(ierr);
if (lu->options.PrintStat) PStatPrint(&lu->options, &stat, &lu->grid); /* Print the statistics. */
PetscStackCall("SuperLU_DIST:PStatFree",PStatFree(&stat));
lu->matsolve_iscalled = PETSC_FALSE;
lu->matmatsolve_iscalled = PETSC_TRUE;
PetscFunctionReturn(0);
}
PetscErrorCode MatSolve_SuperLU_DIST(Mat A,Vec b_mpi,Vec x)
{
Mat_SuperLU_DIST *lu = (Mat_SuperLU_DIST*)A->spptr;
PetscErrorCode ierr;
PetscMPIInt size;
PetscInt m=A->rmap->n,M=A->rmap->N,N=A->cmap->N;
SuperLUStat_t stat;
double berr[1];
PetscScalar *bptr;
PetscInt nrhs=1;
Vec x_seq;
IS iden;
VecScatter scat;
int info; /* SuperLU_Dist info code is ALWAYS an int, even with long long indices */
static PetscBool cite = PETSC_FALSE;
PetscFunctionBegin;
ierr = PetscCitationsRegister("@article{lidemmel03,\n author = {Xiaoye S. Li and James W. Demmel},\n title = {{SuperLU_DIST}: A Scalable Distributed-Memory Sparse Direct\n Solver for Unsymmetric Linear Systems},\n journal = {ACM Trans. Mathematical Software},\n volume = {29},\n number = {2},\n pages = {110-140},\n year = 2003\n}\n",&cite);CHKERRQ(ierr);
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&size);CHKERRQ(ierr);
if (size > 1 && lu->MatInputMode == GLOBAL) {
/* global mat input, convert b to x_seq */
ierr = VecCreateSeq(PETSC_COMM_SELF,N,&x_seq);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,N,0,1,&iden);CHKERRQ(ierr);
ierr = VecScatterCreate(b_mpi,iden,x_seq,iden,&scat);CHKERRQ(ierr);
ierr = ISDestroy(&iden);CHKERRQ(ierr);
ierr = VecScatterBegin(scat,b_mpi,x_seq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scat,b_mpi,x_seq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(x_seq,&bptr);CHKERRQ(ierr);
} else { /* size==1 || distributed mat input */
if (lu->options.SolveInitialized && !lu->matsolve_iscalled) {
/* see comments in MatMatSolve() */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zSolveFinalize",zSolveFinalize(&lu->options, &lu->SOLVEstruct));
#else
PetscStackCall("SuperLU_DIST:dSolveFinalize",dSolveFinalize(&lu->options, &lu->SOLVEstruct));
#endif
lu->options.SolveInitialized = NO;
}
ierr = VecCopy(b_mpi,x);CHKERRQ(ierr);
ierr = VecGetArray(x,&bptr);CHKERRQ(ierr);
}
if (lu->options.Fact != FACTORED) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"SuperLU_DIST options.Fact mush equal FACTORED");
PetscStackCall("SuperLU_DIST:PStatInit",PStatInit(&stat)); /* Initialize the statistics variables. */
if (lu->MatInputMode == GLOBAL) {
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx_ABglobal",pzgssvx_ABglobal(&lu->options,&lu->A_sup,&lu->ScalePermstruct,(doublecomplex*)bptr,M,nrhs,&lu->grid,&lu->LUstruct,berr,&stat,&info));
#else
PetscStackCall("SuperLU_DIST:pdgssvx_ABglobal",pdgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct,bptr,M,nrhs,&lu->grid,&lu->LUstruct,berr,&stat,&info));
#endif
} else { /* distributed mat input */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx",pzgssvx(&lu->options,&lu->A_sup,&lu->ScalePermstruct,(doublecomplex*)bptr,m,nrhs,&lu->grid,&lu->LUstruct,&lu->SOLVEstruct,berr,&stat,&info));
#else
PetscStackCall("SuperLU_DIST:pdgssvx",pdgssvx(&lu->options,&lu->A_sup,&lu->ScalePermstruct,bptr,m,nrhs,&lu->grid,&lu->LUstruct,&lu->SOLVEstruct,berr,&stat,&info));
#endif
}
if (info) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pdgssvx fails, info: %d\n",info);
if (lu->options.PrintStat) PStatPrint(&lu->options, &stat, &lu->grid); /* Print the statistics. */
PetscStackCall("SuperLU_DIST:PStatFree",PStatFree(&stat));
if (size > 1 && lu->MatInputMode == GLOBAL) {
/* convert seq x to mpi x */
ierr = VecRestoreArray(x_seq,&bptr);CHKERRQ(ierr);
ierr = VecScatterBegin(scat,x_seq,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(scat,x_seq,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scat);CHKERRQ(ierr);
ierr = VecDestroy(&x_seq);CHKERRQ(ierr);
} else {
ierr = VecRestoreArray(x,&bptr);CHKERRQ(ierr);
lu->matsolve_iscalled = PETSC_TRUE;
lu->matmatsolve_iscalled = PETSC_FALSE;
}
PetscFunctionReturn(0);
}
PetscErrorCode MatSolve_SuperLU_DIST(Mat A,Vec b_mpi,Vec x)
{
Mat_SuperLU_DIST *lu = (Mat_SuperLU_DIST*)A->spptr;
PetscErrorCode ierr;
PetscMPIInt size;
PetscInt m=A->rmap->n,M=A->rmap->N,N=A->cmap->N;
SuperLUStat_t stat;
double berr[1];
PetscScalar *bptr;
PetscInt nrhs=1;
Vec x_seq;
IS iden;
VecScatter scat;
int info; /* SuperLU_Dist info code is ALWAYS an int, even with long long indices */
PetscFunctionBegin;
ierr = MPI_Comm_size(((PetscObject)A)->comm,&size);CHKERRQ(ierr);
if (size > 1 && lu->MatInputMode == GLOBAL) {
/* global mat input, convert b to x_seq */
ierr = VecCreateSeq(PETSC_COMM_SELF,N,&x_seq);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,N,0,1,&iden);CHKERRQ(ierr);
ierr = VecScatterCreate(b_mpi,iden,x_seq,iden,&scat);CHKERRQ(ierr);
ierr = ISDestroy(&iden);CHKERRQ(ierr);
ierr = VecScatterBegin(scat,b_mpi,x_seq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scat,b_mpi,x_seq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(x_seq,&bptr);CHKERRQ(ierr);
} else { /* size==1 || distributed mat input */
if (lu->options.SolveInitialized && !lu->matsolve_iscalled){
/* see comments in MatMatSolve() */
#if defined(PETSC_USE_COMPLEX)
zSolveFinalize(&lu->options, &lu->SOLVEstruct);
#else
dSolveFinalize(&lu->options, &lu->SOLVEstruct);
#endif
lu->options.SolveInitialized = NO;
}
ierr = VecCopy(b_mpi,x);CHKERRQ(ierr);
ierr = VecGetArray(x,&bptr);CHKERRQ(ierr);
}
if (lu->options.Fact != FACTORED) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"SuperLU_DIST options.Fact mush equal FACTORED");
PStatInit(&stat); /* Initialize the statistics variables. */
if (lu->MatInputMode == GLOBAL) {
#if defined(PETSC_USE_COMPLEX)
pzgssvx_ABglobal(&lu->options,&lu->A_sup,&lu->ScalePermstruct,(doublecomplex*)bptr,M,nrhs,
&lu->grid,&lu->LUstruct,berr,&stat,&info);
#else
pdgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct,bptr,M,nrhs,
&lu->grid,&lu->LUstruct,berr,&stat,&info);
#endif
} else { /* distributed mat input */
#if defined(PETSC_USE_COMPLEX)
pzgssvx(&lu->options,&lu->A_sup,&lu->ScalePermstruct,(doublecomplex*)bptr,m,nrhs,&lu->grid,
&lu->LUstruct,&lu->SOLVEstruct,berr,&stat,&info);
#else
pdgssvx(&lu->options,&lu->A_sup,&lu->ScalePermstruct,bptr,m,nrhs,&lu->grid,
&lu->LUstruct,&lu->SOLVEstruct,berr,&stat,&info);
#endif
}
if (info) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pdgssvx fails, info: %d\n",info);
if (lu->options.PrintStat) {
PStatPrint(&lu->options, &stat, &lu->grid); /* Print the statistics. */
}
PStatFree(&stat);
if (size > 1 && lu->MatInputMode == GLOBAL) {
/* convert seq x to mpi x */
ierr = VecRestoreArray(x_seq,&bptr);CHKERRQ(ierr);
ierr = VecScatterBegin(scat,x_seq,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(scat,x_seq,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scat);CHKERRQ(ierr);
ierr = VecDestroy(&x_seq);CHKERRQ(ierr);
} else {
ierr = VecRestoreArray(x,&bptr);CHKERRQ(ierr);
lu->matsolve_iscalled = PETSC_TRUE;
lu->matmatsolve_iscalled = PETSC_FALSE;
}
PetscFunctionReturn(0);
}
/*! \brief
*
* <pre>
* Purpose
* =======
*
* The driver program PDDRIVE1.
*
* This example illustrates how to use PDGSSVX to
* solve systems with the same A but different right-hand side.
* In this case, we factorize A only once in the first call to
* PDGSSVX, and reuse the following data structures
* in the subsequent call to PDGSSVX:
* ScalePermstruct : DiagScale, R, C, perm_r, perm_c
* LUstruct : Glu_persist, Llu
*
* With MPICH, program may be run by typing:
* mpiexec -n <np> pddrive1 -r <proc rows> -c <proc columns> big.rua
* </pre>
*/
int main(int argc, char *argv[])
{
superlu_dist_options_t options;
SuperLUStat_t stat;
SuperMatrix A;
ScalePermstruct_t ScalePermstruct;
LUstruct_t LUstruct;
SOLVEstruct_t SOLVEstruct;
gridinfo_t grid;
double *berr;
double *b, *xtrue, *b1;
int i, j, m, n;
int nprow, npcol;
int iam, info, ldb, ldx, nrhs;
char **cpp, c, *postfix;
int ii, omp_mpi_level;
FILE *fp, *fopen();
int cpp_defs();
nprow = 1; /* Default process rows. */
npcol = 1; /* Default process columns. */
nrhs = 1; /* Number of right-hand side. */
/* ------------------------------------------------------------
INITIALIZE MPI ENVIRONMENT.
------------------------------------------------------------*/
MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &omp_mpi_level);
/* Parse command line argv[]. */
for (cpp = argv+1; *cpp; ++cpp) {
if ( **cpp == '-' ) {
c = *(*cpp+1);
++cpp;
switch (c) {
case 'h':
printf("Options:\n");
printf("\t-r <int>: process rows (default %d)\n", nprow);
printf("\t-c <int>: process columns (default %d)\n", npcol);
exit(0);
break;
case 'r': nprow = atoi(*cpp);
break;
case 'c': npcol = atoi(*cpp);
break;
}
} else { /* Last arg is considered a filename */
if ( !(fp = fopen(*cpp, "r")) ) {
ABORT("File does not exist");
}
break;
}
}
/* ------------------------------------------------------------
INITIALIZE THE SUPERLU PROCESS GRID.
------------------------------------------------------------*/
superlu_gridinit(MPI_COMM_WORLD, nprow, npcol, &grid);
/* Bail out if I do not belong in the grid. */
iam = grid.iam;
if ( iam >= nprow * npcol ) goto out;
if ( !iam ) {
int v_major, v_minor, v_bugfix;
#ifdef __INTEL_COMPILER
printf("__INTEL_COMPILER is defined\n");
#endif
printf("__STDC_VERSION__ %ld\n", __STDC_VERSION__);
superlu_dist_GetVersionNumber(&v_major, &v_minor, &v_bugfix);
printf("Library version:\t%d.%d.%d\n", v_major, v_minor, v_bugfix);
printf("Input matrix file:\t%s\n", *cpp);
printf("Process grid:\t\t%d X %d\n", (int)grid.nprow, (int)grid.npcol);
fflush(stdout);
}
#if ( VAMPIR>=1 )
VT_traceoff();
#endif
#if ( DEBUGlevel>=1 )
CHECK_MALLOC(iam, "Enter main()");
#endif
for(ii = 0;ii<strlen(*cpp);ii++){
if((*cpp)[ii]=='.'){
postfix = &((*cpp)[ii+1]);
}
}
// printf("%s\n", postfix);
/* ------------------------------------------------------------
GET THE MATRIX FROM FILE AND SETUP THE RIGHT HAND SIDE.
------------------------------------------------------------*/
dcreate_matrix_postfix(&A, nrhs, &b, &ldb, &xtrue, &ldx, fp, postfix, &grid);
if ( !(b1 = doubleMalloc_dist(ldb * nrhs)) )
ABORT("Malloc fails for b1[]");
for (j = 0; j < nrhs; ++j)
for (i = 0; i < ldb; ++i) b1[i+j*ldb] = b[i+j*ldb];
if ( !(berr = doubleMalloc_dist(nrhs)) )
ABORT("Malloc fails for berr[].");
/* ------------------------------------------------------------
WE SOLVE THE LINEAR SYSTEM FOR THE FIRST TIME.
------------------------------------------------------------*/
/* Set the default input options:
options.Fact = DOFACT;
options.Equil = YES;
options.ColPerm = METIS_AT_PLUS_A;
options.RowPerm = LargeDiag_MC64;
options.ReplaceTinyPivot = NO;
options.Trans = NOTRANS;
options.IterRefine = DOUBLE;
options.SolveInitialized = NO;
options.RefineInitialized = NO;
options.PrintStat = YES;
*/
set_default_options_dist(&options);
if (!iam) {
print_sp_ienv_dist(&options);
print_options_dist(&options);
fflush(stdout);
}
m = A.nrow;
n = A.ncol;
/* Initialize ScalePermstruct and LUstruct. */
ScalePermstructInit(m, n, &ScalePermstruct);
LUstructInit(n, &LUstruct);
/* Initialize the statistics variables. */
PStatInit(&stat);
/* Call the linear equation solver. */
pdgssvx(&options, &A, &ScalePermstruct, b, ldb, nrhs, &grid,
&LUstruct, &SOLVEstruct, berr, &stat, &info);
/* Check the accuracy of the solution. */
if ( !iam ) printf("\tSolve the first system:\n");
pdinf_norm_error(iam, ((NRformat_loc *)A.Store)->m_loc,
nrhs, b, ldb, xtrue, ldx, &grid);
PStatPrint(&options, &stat, &grid); /* Print the statistics. */
PStatFree(&stat);
/* ------------------------------------------------------------
NOW WE SOLVE ANOTHER SYSTEM WITH THE SAME A BUT DIFFERENT
RIGHT-HAND SIDE, WE WILL USE THE EXISTING L AND U FACTORS IN
LUSTRUCT OBTAINED FROM A PREVIOUS FATORIZATION.
------------------------------------------------------------*/
options.Fact = FACTORED; /* Indicate the factored form of A is supplied. */
PStatInit(&stat); /* Initialize the statistics variables. */
pdgssvx(&options, &A, &ScalePermstruct, b1, ldb, nrhs, &grid,
&LUstruct, &SOLVEstruct, berr, &stat, &info);
/* Check the accuracy of the solution. */
if ( !iam ) printf("\tSolve the system with a different B:\n");
pdinf_norm_error(iam, ((NRformat_loc *)A.Store)->m_loc,
nrhs, b1, ldb, xtrue, ldx, &grid);
PStatPrint(&options, &stat, &grid); /* Print the statistics. */
/* ------------------------------------------------------------
DEALLOCATE STORAGE.
------------------------------------------------------------*/
PStatFree(&stat);
Destroy_CompRowLoc_Matrix_dist(&A);
ScalePermstructFree(&ScalePermstruct);
Destroy_LU(n, &grid, &LUstruct);
LUstructFree(&LUstruct);
if ( options.SolveInitialized ) {
dSolveFinalize(&options, &SOLVEstruct);
}
SUPERLU_FREE(b);
SUPERLU_FREE(b1);
SUPERLU_FREE(xtrue);
SUPERLU_FREE(berr);
/* ------------------------------------------------------------
RELEASE THE SUPERLU PROCESS GRID.
------------------------------------------------------------*/
out:
superlu_gridexit(&grid);
/* ------------------------------------------------------------
TERMINATES THE MPI EXECUTION ENVIRONMENT.
------------------------------------------------------------*/
MPI_Finalize();
#if ( DEBUGlevel>=1 )
CHECK_MALLOC(iam, "Exit main()");
#endif
}