PetscErrorCode MatIsSymmetric_IS(Mat A,PetscReal tol,PetscBool *flg)
{
PetscErrorCode ierr;
Mat_IS *matis = (Mat_IS*)A->data;
PetscBool local_sym;
PetscFunctionBegin;
ierr = MatIsSymmetric(matis->A,tol,&local_sym);CHKERRQ(ierr);
ierr = MPIU_Allreduce(&local_sym,flg,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)A));CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode PCSetUp_SupportGraph(PC pc)
{
PC_SupportGraph *sg = (PC_SupportGraph*)pc->data;
PetscBool isSym;
PetscErrorCode ierr;
/*
Vec diag;
PetscInt n,i;
PetscScalar *x;
PetscBool zeroflag = PETSC_FALSE;
*/
PetscFunctionBegin;
if (!pc->setupcalled) {
ierr = MatIsSymmetric(pc->pmat, 1.0e-9, &isSym);CHKERRQ(ierr);
if (!isSym) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_ARG_WRONG,"matrix must be symmetric");
/* note that maxCong is being updated */
ierr = AugmentedLowStretchSpanningTree(pc->pmat, &sg->pre, sg->augment, sg->tol, sg->maxCong);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode MatFactorNumeric_PaStiX(Mat F,Mat A,const MatFactorInfo *info)
{
Mat_Pastix *lu =(Mat_Pastix*)(F)->spptr;
Mat *tseq;
PetscErrorCode ierr = 0;
PetscInt icntl;
PetscInt M=A->rmap->N;
PetscBool valOnly,flg, isSym;
Mat F_diag;
IS is_iden;
Vec b;
IS isrow;
PetscBool isSeqAIJ,isSeqSBAIJ,isMPIAIJ;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isSeqAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&isMPIAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQSBAIJ,&isSeqSBAIJ);CHKERRQ(ierr);
if (lu->matstruc == DIFFERENT_NONZERO_PATTERN) {
(F)->ops->solve = MatSolve_PaStiX;
/* Initialize a PASTIX instance */
ierr = MPI_Comm_dup(PetscObjectComm((PetscObject)A),&(lu->pastix_comm));CHKERRQ(ierr);
ierr = MPI_Comm_rank(lu->pastix_comm, &lu->commRank);CHKERRQ(ierr);
ierr = MPI_Comm_size(lu->pastix_comm, &lu->commSize);CHKERRQ(ierr);
/* Set pastix options */
lu->iparm[IPARM_MODIFY_PARAMETER] = API_NO;
lu->iparm[IPARM_START_TASK] = API_TASK_INIT;
lu->iparm[IPARM_END_TASK] = API_TASK_INIT;
lu->rhsnbr = 1;
/* Call to set default pastix options */
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
ierr = PetscOptionsBegin(PetscObjectComm((PetscObject)A),((PetscObject)A)->prefix,"PaStiX Options","Mat");CHKERRQ(ierr);
icntl = -1;
lu->iparm[IPARM_VERBOSE] = API_VERBOSE_NOT;
ierr = PetscOptionsInt("-mat_pastix_verbose","iparm[IPARM_VERBOSE] : level of printing (0 to 2)","None",lu->iparm[IPARM_VERBOSE],&icntl,&flg);CHKERRQ(ierr);
if ((flg && icntl >= 0) || PetscLogPrintInfo) {
lu->iparm[IPARM_VERBOSE] = icntl;
}
icntl=-1;
ierr = PetscOptionsInt("-mat_pastix_threadnbr","iparm[IPARM_THREAD_NBR] : Number of thread by MPI node","None",lu->iparm[IPARM_THREAD_NBR],&icntl,&flg);CHKERRQ(ierr);
if ((flg && icntl > 0)) {
lu->iparm[IPARM_THREAD_NBR] = icntl;
}
PetscOptionsEnd();
valOnly = PETSC_FALSE;
} else {
if (isSeqAIJ || isMPIAIJ) {
ierr = PetscFree(lu->colptr);CHKERRQ(ierr);
ierr = PetscFree(lu->row);CHKERRQ(ierr);
ierr = PetscFree(lu->val);CHKERRQ(ierr);
valOnly = PETSC_FALSE;
} else valOnly = PETSC_TRUE;
}
lu->iparm[IPARM_MATRIX_VERIFICATION] = API_YES;
/* convert mpi A to seq mat A */
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&tseq);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = MatConvertToCSC(*tseq,valOnly, &lu->n, &lu->colptr, &lu->row, &lu->val);CHKERRQ(ierr);
ierr = MatIsSymmetric(*tseq,0.0,&isSym);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&tseq);CHKERRQ(ierr);
if (!lu->perm) {
ierr = PetscMalloc1((lu->n),&(lu->perm));CHKERRQ(ierr);
ierr = PetscMalloc1((lu->n),&(lu->invp));CHKERRQ(ierr);
}
if (isSym) {
/* On symmetric matrix, LLT */
lu->iparm[IPARM_SYM] = API_SYM_YES;
lu->iparm[IPARM_FACTORIZATION] = API_FACT_LDLT;
} else {
/* On unsymmetric matrix, LU */
lu->iparm[IPARM_SYM] = API_SYM_NO;
lu->iparm[IPARM_FACTORIZATION] = API_FACT_LU;
}
/*----------------*/
if (lu->matstruc == DIFFERENT_NONZERO_PATTERN) {
if (!(isSeqAIJ || isSeqSBAIJ)) {
/* PaStiX only supports centralized rhs. Create scatter scat_rhs for repeated use in MatSolve() */
ierr = VecCreateSeq(PETSC_COMM_SELF,A->cmap->N,&lu->b_seq);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,A->cmap->N,0,1,&is_iden);CHKERRQ(ierr);
ierr = MatCreateVecs(A,NULL,&b);CHKERRQ(ierr);
ierr = VecScatterCreate(b,is_iden,lu->b_seq,is_iden,&lu->scat_rhs);CHKERRQ(ierr);
ierr = VecScatterCreate(lu->b_seq,is_iden,b,is_iden,&lu->scat_sol);CHKERRQ(ierr);
ierr = ISDestroy(&is_iden);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
}
lu->iparm[IPARM_START_TASK] = API_TASK_ORDERING;
lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]);
} else {
lu->iparm[IPARM_START_TASK] = API_TASK_NUMFACT;
lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]);
}
if (lu->commSize > 1) {
if ((F)->factortype == MAT_FACTOR_LU) {
F_diag = ((Mat_MPIAIJ*)(F)->data)->A;
} else {
F_diag = ((Mat_MPISBAIJ*)(F)->data)->A;
}
F_diag->assembled = PETSC_TRUE;
}
(F)->assembled = PETSC_TRUE;
lu->matstruc = SAME_NONZERO_PATTERN;
lu->CleanUpPastix = PETSC_TRUE;
PetscFunctionReturn(0);
}
/*
convert Petsc seqaij matrix to CSC: colptr[n], row[nz], val[nz]
input:
A - matrix in seqaij or mpisbaij (bs=1) format
valOnly - FALSE: spaces are allocated and values are set for the CSC
TRUE: Only fill values
output:
n - Size of the matrix
colptr - Index of first element of each column in row and val
row - Row of each element of the matrix
values - Value of each element of the matrix
*/
PetscErrorCode MatConvertToCSC(Mat A,PetscBool valOnly,PetscInt *n,PetscInt **colptr,PetscInt **row,PetscScalar **values)
{
Mat_SeqAIJ *aa = (Mat_SeqAIJ*)A->data;
PetscInt *rowptr = aa->i;
PetscInt *col = aa->j;
PetscScalar *rvalues = aa->a;
PetscInt m = A->rmap->N;
PetscInt nnz;
PetscInt i,j, k;
PetscInt base = 1;
PetscInt idx;
PetscErrorCode ierr;
PetscInt colidx;
PetscInt *colcount;
PetscBool isSBAIJ;
PetscBool isSeqSBAIJ;
PetscBool isMpiSBAIJ;
PetscBool isSym;
PetscBool flg;
PetscInt icntl;
PetscInt verb;
PetscInt check;
PetscFunctionBegin;
ierr = MatIsSymmetric(A,0.0,&isSym);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSBAIJ,&isSBAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQSBAIJ,&isSeqSBAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPISBAIJ,&isMpiSBAIJ);CHKERRQ(ierr);
*n = A->cmap->N;
/* PaStiX only needs triangular matrix if matrix is symmetric
*/
if (isSym && !(isSBAIJ || isSeqSBAIJ || isMpiSBAIJ)) nnz = (aa->nz - *n)/2 + *n;
else nnz = aa->nz;
if (!valOnly) {
ierr = PetscMalloc(((*n)+1) *sizeof(PetscInt) ,colptr);CHKERRQ(ierr);
ierr = PetscMalloc(nnz *sizeof(PetscInt) ,row);CHKERRQ(ierr);
ierr = PetscMalloc1(nnz ,values);CHKERRQ(ierr);
if (isSBAIJ || isSeqSBAIJ || isMpiSBAIJ) {
ierr = PetscMemcpy (*colptr, rowptr, ((*n)+1)*sizeof(PetscInt));CHKERRQ(ierr);
for (i = 0; i < *n+1; i++) (*colptr)[i] += base;
ierr = PetscMemcpy (*row, col, (nnz)*sizeof(PetscInt));CHKERRQ(ierr);
for (i = 0; i < nnz; i++) (*row)[i] += base;
ierr = PetscMemcpy (*values, rvalues, (nnz)*sizeof(PetscScalar));CHKERRQ(ierr);
} else {
ierr = PetscMalloc1((*n),&colcount);CHKERRQ(ierr);
for (i = 0; i < m; i++) colcount[i] = 0;
/* Fill-in colptr */
for (i = 0; i < m; i++) {
for (j = rowptr[i]; j < rowptr[i+1]; j++) {
if (!isSym || col[j] <= i) colcount[col[j]]++;
}
}
(*colptr)[0] = base;
for (j = 0; j < *n; j++) {
(*colptr)[j+1] = (*colptr)[j] + colcount[j];
/* in next loop we fill starting from (*colptr)[colidx] - base */
colcount[j] = -base;
}
/* Fill-in rows and values */
for (i = 0; i < m; i++) {
for (j = rowptr[i]; j < rowptr[i+1]; j++) {
if (!isSym || col[j] <= i) {
colidx = col[j];
idx = (*colptr)[colidx] + colcount[colidx];
(*row)[idx] = i + base;
(*values)[idx] = rvalues[j];
colcount[colidx]++;
}
}
}
ierr = PetscFree(colcount);CHKERRQ(ierr);
}
} else {
/* Fill-in only values */
for (i = 0; i < m; i++) {
for (j = rowptr[i]; j < rowptr[i+1]; j++) {
colidx = col[j];
if ((isSBAIJ || isSeqSBAIJ || isMpiSBAIJ) ||!isSym || col[j] <= i) {
/* look for the value to fill */
for (k = (*colptr)[colidx] - base; k < (*colptr)[colidx + 1] - base; k++) {
if (((*row)[k]-base) == i) {
(*values)[k] = rvalues[j];
break;
}
}
/* data structure of sparse matrix has changed */
if (k == (*colptr)[colidx + 1] - base) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"overflow on k %D",k);
}
}
}
}
icntl =-1;
check = 0;
ierr = PetscOptionsGetInt(((PetscObject) A)->prefix, "-mat_pastix_check", &icntl, &flg);CHKERRQ(ierr);
if ((flg && icntl >= 0) || PetscLogPrintInfo) check = icntl;
if (check == 1) {
PetscScalar *tmpvalues;
PetscInt *tmprows,*tmpcolptr;
tmpvalues = (PetscScalar*)malloc(nnz*sizeof(PetscScalar)); if (!tmpvalues) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Unable to allocate memory");
tmprows = (PetscInt*) malloc(nnz*sizeof(PetscInt)); if (!tmprows) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Unable to allocate memory");
tmpcolptr = (PetscInt*) malloc((*n+1)*sizeof(PetscInt)); if (!tmpcolptr) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Unable to allocate memory");
ierr = PetscMemcpy(tmpcolptr,*colptr,(*n+1)*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(tmprows,*row,nnz*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(tmpvalues,*values,nnz*sizeof(PetscScalar));CHKERRQ(ierr);
ierr = PetscFree(*row);CHKERRQ(ierr);
ierr = PetscFree(*values);CHKERRQ(ierr);
icntl=-1;
verb = API_VERBOSE_NOT;
/* "iparm[IPARM_VERBOSE] : level of printing (0 to 2)" */
ierr = PetscOptionsGetInt(((PetscObject) A)->prefix, "-mat_pastix_verbose", &icntl, &flg);CHKERRQ(ierr);
if ((flg && icntl >= 0) || PetscLogPrintInfo) verb = icntl;
PASTIX_CHECKMATRIX(MPI_COMM_WORLD,verb,((isSym != 0) ? API_SYM_YES : API_SYM_NO),API_YES,*n,&tmpcolptr,&tmprows,(PastixScalar**)&tmpvalues,NULL,1);
ierr = PetscMemcpy(*colptr,tmpcolptr,(*n+1)*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMalloc1(((*colptr)[*n]-1),row);CHKERRQ(ierr);
ierr = PetscMemcpy(*row,tmprows,((*colptr)[*n]-1)*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMalloc1(((*colptr)[*n]-1),values);CHKERRQ(ierr);
ierr = PetscMemcpy(*values,tmpvalues,((*colptr)[*n]-1)*sizeof(PetscScalar));CHKERRQ(ierr);
free(tmpvalues);
free(tmprows);
free(tmpcolptr);
}
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Vec x, b, u, u2;
Mat A, L, PL;
KSP ksp;
// PC pc;
PetscReal norm;
PetscInt m1 = 5, m = 8,n = 7, p=1, d=0, dd, its;
PetscErrorCode ierr;
// PetscTruth user_defined_pc, user_defined_mat;
ierr = PetscInitialize(&argc,&args,(char *)0,help); CHKERRQ(ierr);
ierr = fftw_import_system_wisdom(); CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m1",&m1,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-d",&d,PETSC_NULL);CHKERRQ(ierr);
if (n == 1) dd = 1;
else if (p == 1) dd = 2;
else dd = 3;
int dim[] = { m, n, p };
// ierr = VecCreate(PETSC_COMM_WORLD, &u2);CHKERRQ(ierr);
// ierr = VecSetSizes(u2, PETSC_DECIDE, m * n);CHKERRQ(ierr);
// ierr = VecSetFromOptions(u2);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_WORLD, m * n * p, &u); CHKERRQ(ierr);
ierr = VecDuplicate(u, &b);CHKERRQ(ierr);
ierr = VecDuplicate(u, &u2);CHKERRQ(ierr);
ierr = VecDuplicate(u, &x);CHKERRQ(ierr);
ierr = MatCreateCheb(PETSC_COMM_WORLD, dd, d, dim, FFTW_ESTIMATE, x, b, &A); CHKERRQ(ierr);
ierr = MatPoissonCreate(PETSC_COMM_WORLD, dd, dim, FFTW_ESTIMATE, x, b, &L); CHKERRQ(ierr);
ierr = AssemblePoissonPC2(m, n, &PL); CHKERRQ(ierr);
if (false) {
PetscTruth flag;
ierr = MatIsSymmetric(PL, 1e-4, &flag); CHKERRQ(ierr);
printf("Symmetric? %d\n", flag);
}
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,L,PL,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
// 3-D solution function
double *uu, *uu2;
ierr = VecGetArray(u, &uu); CHKERRQ(ierr);
ierr = VecGetArray(u2, &uu2); CHKERRQ(ierr);
for (int i=0; i < m; i++) {
double x = (m==1) ? 0 : cos (i * PI / (m-1));
for (int j=0; j < n; j++) {
double y = (n==1) ? 0 : cos (j * PI / (n-1));
for (int k=0; k < p; k++) {
// double z = (p==1) ? 0 : cos (k * PI / (p-1));
int ix = (i*n + j) * p + k;
uu[ix] = cos(0.5 * PI * x) * cos (0.5 * PI * y);
uu2[ix] = 0.5 * PI * PI * uu[ix];
// uu[ix] = pow(x, 4) * pow(y, 3) + pow(y, 5);
// uu2[ix] = -(4 * 3 * pow(x, 2) * pow(y, 3) + 3 * 2 * pow(x,4) * y + 5 * 4 * pow(y, 3));
//uu[ix] =
}
}
}
ierr = VecRestoreArray(u, &uu); CHKERRQ(ierr);
ierr = VecRestoreArray(u2, &uu2); CHKERRQ(ierr);
ierr = VecCopy(u2, b); CHKERRQ(ierr);
#if 0
ierr = MatMult(PL, u, x); CHKERRQ(ierr);
//ierr = VecCopy(b, x); CHKERRQ(ierr);
ierr = VecView(x, PETSC_VIEWER_STDOUT_SELF); CHKERRQ(ierr);
printf("\n");
ierr = VecView(u2, PETSC_VIEWER_STDOUT_SELF); CHKERRQ(ierr);
printf("\n");
ierr = VecAXPY(x, -1.0, u2);CHKERRQ(ierr);
ierr = VecNorm(x, NORM_INFINITY, &norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %A\n",norm);CHKERRQ(ierr);
#endif
// ierr = MatMult(L, u, b); CHKERRQ(ierr);
#if BC
{ // Set up boundary conditions
double *uu, *bb;
const int m = dim[0], n = dim[1];
ierr = VecGetArray(u, &uu); CHKERRQ(ierr);
ierr = VecGetArray(b, &bb); CHKERRQ(ierr);
for (int i = 0; i < m; i++) {
int ix0 = i * n + 0;
int ixn = i * n + n - 1;
bb[ix0] = uu[ix0];
bb[ixn] = uu[ixn];
}
for (int j = 0; j < n; j++) {
int ix0 = 1 * n + j;
int ixn = (m-2) * n + j;
bb[ix0] = uu[ix0];
bb[ixn] = uu[ixn];
}
ierr = VecRestoreArray(u, &uu); CHKERRQ(ierr);
ierr = VecRestoreArray(b, &bb); CHKERRQ(ierr);
}
#endif
#if SOLVE
ierr = VecSet(x, 0.0); CHKERRQ(ierr);
ierr = KSPSolve(ksp, b, x); CHKERRQ(ierr);
/* ierr = VecView(b, PETSC_VIEWER_STDOUT_SELF); CHKERRQ(ierr); */
/* printf("\n"); */
/* ierr = VecView(u, PETSC_VIEWER_STDOUT_SELF); CHKERRQ(ierr); */
/* printf("\n"); */
/* ierr = VecView(x, PETSC_VIEWER_STDOUT_SELF); CHKERRQ(ierr); */
/* printf("\n"); */
ierr = VecAXPY(x, -1.0, u); CHKERRQ(ierr);
ierr = VecNorm(x, NORM_INFINITY, &norm); CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp, &its); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Norm of error %A iterations %D\n", norm, its); CHKERRQ(ierr);
// ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %A\n",norm);CHKERRQ(ierr);
#endif
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = KSPDestroy(ksp);CHKERRQ(ierr);
ierr = VecDestroy(u);CHKERRQ(ierr); ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(b);CHKERRQ(ierr); ierr = MatDestroy(A);CHKERRQ(ierr);
/* if (user_defined_pc) { */
/* ierr = SampleShellPCDestroy(shell);CHKERRQ(ierr); */
/* } */
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}