int main(int argc,char **args)
{
Mat BAIJ,SBAIJ,*subBAIJ,*subSBAIJ;
PetscViewer viewer;
char file[PETSC_MAX_PATH_LEN];
PetscBool flg;
PetscErrorCode ierr;
PetscInt n = 2,issize,M,N;
PetscMPIInt rank;
IS isrow,iscol,irow[n],icol[n];
PetscInitialize(&argc,&args,(char*)0,help);
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&BAIJ);CHKERRQ(ierr);
ierr = MatSetType(BAIJ,MATMPIBAIJ);CHKERRQ(ierr);
ierr = MatLoad(BAIJ,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&SBAIJ);CHKERRQ(ierr);
ierr = MatSetType(SBAIJ,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatLoad(SBAIJ,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = MatGetSize(BAIJ,&M,&N);CHKERRQ(ierr);
issize = M/4;
ierr = ISCreateStride(PETSC_COMM_SELF,issize,0,1,&isrow);CHKERRQ(ierr);
irow[0] = irow[1] = isrow;
issize = N/2;
ierr = ISCreateStride(PETSC_COMM_SELF,issize,0,1,&iscol);CHKERRQ(ierr);
icol[0] = icol[1] = iscol;
ierr = MatGetSubMatrices(BAIJ,n,irow,icol,MAT_INITIAL_MATRIX,&subBAIJ);CHKERRQ(ierr);
/* irow and icol must be same for SBAIJ matrices! */
icol[0] = icol[1] = isrow;
ierr = MatGetSubMatrices(SBAIJ,n,irow,icol,MAT_INITIAL_MATRIX,&subSBAIJ);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (!rank) {
ierr = MatView(subBAIJ[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatView(subSBAIJ[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
/* Free data structures */
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
ierr = MatDestroyMatrices(n,&subBAIJ);CHKERRQ(ierr);
ierr = MatDestroyMatrices(n,&subSBAIJ);CHKERRQ(ierr);
ierr = MatDestroy(&BAIJ);CHKERRQ(ierr);
ierr = MatDestroy(&SBAIJ);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat BAIJ,SBAIJ,*subBAIJ,*subSBAIJ;
PetscViewer viewer;
char file[PETSC_MAX_PATH_LEN];
PetscBool flg;
PetscErrorCode ierr;
PetscInt n = 2,issize;
PetscMPIInt rank;
IS is,iss[2];
PetscInitialize(&argc,&args,(char*)0,help);
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&BAIJ);CHKERRQ(ierr);
ierr = MatSetType(BAIJ,MATMPIBAIJ);CHKERRQ(ierr);
ierr = MatLoad(BAIJ,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&SBAIJ);CHKERRQ(ierr);
ierr = MatSetType(SBAIJ,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatLoad(SBAIJ,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = MatGetSize(BAIJ,&issize,0);CHKERRQ(ierr);
issize = 9;
ierr = ISCreateStride(PETSC_COMM_SELF,issize,0,1,&is);CHKERRQ(ierr);
iss[0] = is;iss[1] = is;
ierr = MatGetSubMatrices(BAIJ,n,iss,iss,MAT_INITIAL_MATRIX,&subBAIJ);CHKERRQ(ierr);
ierr = MatGetSubMatrices(SBAIJ,n,iss,iss,MAT_INITIAL_MATRIX,&subSBAIJ);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
#if defined(PETSC_USE_SOCKET_VIEWER)
if (!rank) {
ierr = MatView(subBAIJ[0],PETSC_VIEWER_SOCKET_SELF);CHKERRQ(ierr);
ierr = MatView(subSBAIJ[0],PETSC_VIEWER_SOCKET_SELF);CHKERRQ(ierr);
}
#endif
/* Free data structures */
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatDestroyMatrices(n,&subBAIJ);CHKERRQ(ierr);
ierr = MatDestroyMatrices(n,&subSBAIJ);CHKERRQ(ierr);
ierr = MatDestroy(&BAIJ);CHKERRQ(ierr);
ierr = MatDestroy(&SBAIJ);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode PCGAMGProlongator_Classical_Standard(PC pc, const Mat A, const Mat G, PetscCoarsenData *agg_lists,Mat *P)
{
PetscErrorCode ierr;
Mat *lA;
Vec lv,v,cv;
PetscScalar *lcid;
IS lis;
PetscInt fs,fe,cs,ce,nl,i,j,k,li,lni,ci;
VecScatter lscat;
PetscInt fn,cn,cid,c_indx;
PetscBool iscoarse;
PetscScalar c_scalar;
const PetscScalar *vcol;
const PetscInt *icol;
const PetscInt *gidx;
PetscInt ncols;
PetscInt *lsparse,*gsparse;
MatType mtype;
PetscInt maxcols;
PetscReal diag,jdiag,jwttotal;
PetscScalar *pvcol,vi;
PetscInt *picol;
PetscInt pncols;
PetscScalar *pcontrib,pentry,pjentry;
/* PC_MG *mg = (PC_MG*)pc->data; */
/* PC_GAMG *gamg = (PC_GAMG*)mg->innerctx; */
PetscFunctionBegin;
ierr = MatGetOwnershipRange(A,&fs,&fe);CHKERRQ(ierr);
fn = fe-fs;
ierr = MatGetVecs(A,NULL,&v);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,fe-fs,fs,1,&lis);CHKERRQ(ierr);
/* increase the overlap by two to get neighbors of neighbors */
ierr = MatIncreaseOverlap(A,1,&lis,2);CHKERRQ(ierr);
ierr = ISSort(lis);CHKERRQ(ierr);
/* get the local part of A */
ierr = MatGetSubMatrices(A,1,&lis,&lis,MAT_INITIAL_MATRIX,&lA);CHKERRQ(ierr);
/* build the scatter out of it */
ierr = ISGetLocalSize(lis,&nl);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,nl,&lv);CHKERRQ(ierr);
ierr = VecScatterCreate(v,lis,lv,NULL,&lscat);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&lsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&gsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*nl,&pcontrib);CHKERRQ(ierr);
/* create coarse vector */
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse);CHKERRQ(ierr);
if (!iscoarse) {
cn++;
}
}
ierr = VecCreateMPI(PetscObjectComm((PetscObject)A),cn,PETSC_DECIDE,&cv);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(cv,&cs,&ce);CHKERRQ(ierr);
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
if (!iscoarse) {
cid = cs+cn;
cn++;
} else {
cid = -1;
}
*(PetscInt*)&c_scalar = cid;
c_indx = fs+i;
ierr = VecSetValues(v,1,&c_indx,&c_scalar,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecScatterBegin(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* count to preallocate the prolongator */
ierr = ISGetIndices(lis,&gidx);CHKERRQ(ierr);
ierr = VecGetArray(lv,&lcid);CHKERRQ(ierr);
maxcols = 0;
/* count the number of unique contributing coarse cells for each fine */
for (i=0;i<nl;i++) {
pcontrib[i] = 0.;
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
if (gidx[i] >= fs && gidx[i] < fe) {
li = gidx[i] - fs;
lsparse[li] = 0;
gsparse[li] = 0;
cid = *(PetscInt*)&(lcid[i]);
if (cid >= 0) {
lsparse[li] = 1;
} else {
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0) {
pcontrib[icol[j]] = 1.;
} else {
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0) {
pcontrib[icol[k]] = 1.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
}
}
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0 && pcontrib[icol[j]] != 0.) {
lni = *(PetscInt*)&(lcid[icol[j]]);
if (lni >= cs && lni < ce) {
lsparse[li]++;
} else {
gsparse[li]++;
}
pcontrib[icol[j]] = 0.;
} else {
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && pcontrib[icol[k]] != 0.) {
lni = *(PetscInt*)&(lcid[icol[k]]);
if (lni >= cs && lni < ce) {
lsparse[li]++;
} else {
gsparse[li]++;
}
pcontrib[icol[k]] = 0.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
}
}
}
if (lsparse[li] + gsparse[li] > maxcols) maxcols = lsparse[li]+gsparse[li];
}
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
ierr = PetscMalloc(sizeof(PetscInt)*maxcols,&picol);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*maxcols,&pvcol);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)A),P);CHKERRQ(ierr);
ierr = MatGetType(A,&mtype);CHKERRQ(ierr);
ierr = MatSetType(*P,mtype);CHKERRQ(ierr);
ierr = MatSetSizes(*P,fn,cn,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*P,0,lsparse,0,gsparse);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*P,0,lsparse);CHKERRQ(ierr);
for (i=0;i<nl;i++) {
diag = 0.;
if (gidx[i] >= fs && gidx[i] < fe) {
li = gidx[i] - fs;
pncols=0;
cid = *(PetscInt*)&(lcid[i]);
if (cid >= 0) {
pncols = 1;
picol[0] = cid;
pvcol[0] = 1.;
} else {
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
pentry = vcol[j];
if (*(PetscInt*)&(lcid[icol[j]]) >= 0) {
/* coarse neighbor */
pcontrib[icol[j]] += pentry;
} else if (icol[j] != i) {
/* the neighbor is a strongly connected fine node */
ci = icol[j];
vi = vcol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
jwttotal=0.;
jdiag = 0.;
for (k=0;k<ncols;k++) {
if (ci == icol[k]) {
jdiag = PetscRealPart(vcol[k]);
}
}
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && jdiag*PetscRealPart(vcol[k]) < 0.) {
pjentry = vcol[k];
jwttotal += PetscRealPart(pjentry);
}
}
if (jwttotal != 0.) {
jwttotal = PetscRealPart(vi)/jwttotal;
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && jdiag*PetscRealPart(vcol[k]) < 0.) {
pjentry = vcol[k]*jwttotal;
pcontrib[icol[k]] += pjentry;
}
}
} else {
diag += PetscRealPart(vi);
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
} else {
diag += PetscRealPart(vcol[j]);
}
}
if (diag != 0.) {
diag = 1./diag;
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0 && pcontrib[icol[j]] != 0.) {
/* the neighbor is a coarse node */
if (PetscAbsScalar(pcontrib[icol[j]]) > 0.0) {
lni = *(PetscInt*)&(lcid[icol[j]]);
pvcol[pncols] = -pcontrib[icol[j]]*diag;
picol[pncols] = lni;
pncols++;
}
pcontrib[icol[j]] = 0.;
} else {
/* the neighbor is a strongly connected fine node */
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && pcontrib[icol[k]] != 0.) {
if (PetscAbsScalar(pcontrib[icol[k]]) > 0.0) {
lni = *(PetscInt*)&(lcid[icol[k]]);
pvcol[pncols] = -pcontrib[icol[k]]*diag;
picol[pncols] = lni;
pncols++;
}
pcontrib[icol[k]] = 0.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
pcontrib[icol[j]] = 0.;
}
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
}
ci = gidx[i];
li = gidx[i] - fs;
if (pncols > 0) {
ierr = MatSetValues(*P,1,&ci,pncols,picol,pvcol,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = ISRestoreIndices(lis,&gidx);CHKERRQ(ierr);
ierr = VecRestoreArray(lv,&lcid);CHKERRQ(ierr);
ierr = PetscFree(pcontrib);CHKERRQ(ierr);
ierr = PetscFree(picol);CHKERRQ(ierr);
ierr = PetscFree(pvcol);CHKERRQ(ierr);
ierr = PetscFree(lsparse);CHKERRQ(ierr);
ierr = PetscFree(gsparse);CHKERRQ(ierr);
ierr = ISDestroy(&lis);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lA);CHKERRQ(ierr);
ierr = VecDestroy(&lv);CHKERRQ(ierr);
ierr = VecDestroy(&cv);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
ierr = VecScatterDestroy(&lscat);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Mat Pold;
ierr = PCGAMGProlongator_Classical(pc,A,G,agg_lists,&Pold);CHKERRQ(ierr);
ierr = MatView(Pold,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatView(*P,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&Pold);CHKERRQ(ierr);
*/
PetscFunctionReturn(0);
}
PetscErrorCode MatFactorNumeric_PaStiX(Mat F,Mat A,const MatFactorInfo *info)
{
Mat_Pastix *lu =(Mat_Pastix*)(F)->spptr;
Mat *tseq;
PetscErrorCode ierr = 0;
PetscInt icntl;
PetscInt M=A->rmap->N;
PetscBool valOnly,flg, isSym;
Mat F_diag;
IS is_iden;
Vec b;
IS isrow;
PetscBool isSeqAIJ,isSeqSBAIJ,isMPIAIJ;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isSeqAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&isMPIAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQSBAIJ,&isSeqSBAIJ);CHKERRQ(ierr);
if (lu->matstruc == DIFFERENT_NONZERO_PATTERN) {
(F)->ops->solve = MatSolve_PaStiX;
/* Initialize a PASTIX instance */
ierr = MPI_Comm_dup(PetscObjectComm((PetscObject)A),&(lu->pastix_comm));CHKERRQ(ierr);
ierr = MPI_Comm_rank(lu->pastix_comm, &lu->commRank);CHKERRQ(ierr);
ierr = MPI_Comm_size(lu->pastix_comm, &lu->commSize);CHKERRQ(ierr);
/* Set pastix options */
lu->iparm[IPARM_MODIFY_PARAMETER] = API_NO;
lu->iparm[IPARM_START_TASK] = API_TASK_INIT;
lu->iparm[IPARM_END_TASK] = API_TASK_INIT;
lu->rhsnbr = 1;
/* Call to set default pastix options */
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
ierr = PetscOptionsBegin(PetscObjectComm((PetscObject)A),((PetscObject)A)->prefix,"PaStiX Options","Mat");CHKERRQ(ierr);
icntl = -1;
lu->iparm[IPARM_VERBOSE] = API_VERBOSE_NOT;
ierr = PetscOptionsInt("-mat_pastix_verbose","iparm[IPARM_VERBOSE] : level of printing (0 to 2)","None",lu->iparm[IPARM_VERBOSE],&icntl,&flg);CHKERRQ(ierr);
if ((flg && icntl >= 0) || PetscLogPrintInfo) {
lu->iparm[IPARM_VERBOSE] = icntl;
}
icntl=-1;
ierr = PetscOptionsInt("-mat_pastix_threadnbr","iparm[IPARM_THREAD_NBR] : Number of thread by MPI node","None",lu->iparm[IPARM_THREAD_NBR],&icntl,&flg);CHKERRQ(ierr);
if ((flg && icntl > 0)) {
lu->iparm[IPARM_THREAD_NBR] = icntl;
}
PetscOptionsEnd();
valOnly = PETSC_FALSE;
} else {
if (isSeqAIJ || isMPIAIJ) {
ierr = PetscFree(lu->colptr);CHKERRQ(ierr);
ierr = PetscFree(lu->row);CHKERRQ(ierr);
ierr = PetscFree(lu->val);CHKERRQ(ierr);
valOnly = PETSC_FALSE;
} else valOnly = PETSC_TRUE;
}
lu->iparm[IPARM_MATRIX_VERIFICATION] = API_YES;
/* convert mpi A to seq mat A */
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&tseq);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = MatConvertToCSC(*tseq,valOnly, &lu->n, &lu->colptr, &lu->row, &lu->val);CHKERRQ(ierr);
ierr = MatIsSymmetric(*tseq,0.0,&isSym);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&tseq);CHKERRQ(ierr);
if (!lu->perm) {
ierr = PetscMalloc1((lu->n),&(lu->perm));CHKERRQ(ierr);
ierr = PetscMalloc1((lu->n),&(lu->invp));CHKERRQ(ierr);
}
if (isSym) {
/* On symmetric matrix, LLT */
lu->iparm[IPARM_SYM] = API_SYM_YES;
lu->iparm[IPARM_FACTORIZATION] = API_FACT_LDLT;
} else {
/* On unsymmetric matrix, LU */
lu->iparm[IPARM_SYM] = API_SYM_NO;
lu->iparm[IPARM_FACTORIZATION] = API_FACT_LU;
}
/*----------------*/
if (lu->matstruc == DIFFERENT_NONZERO_PATTERN) {
if (!(isSeqAIJ || isSeqSBAIJ)) {
/* PaStiX only supports centralized rhs. Create scatter scat_rhs for repeated use in MatSolve() */
ierr = VecCreateSeq(PETSC_COMM_SELF,A->cmap->N,&lu->b_seq);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,A->cmap->N,0,1,&is_iden);CHKERRQ(ierr);
ierr = MatCreateVecs(A,NULL,&b);CHKERRQ(ierr);
ierr = VecScatterCreate(b,is_iden,lu->b_seq,is_iden,&lu->scat_rhs);CHKERRQ(ierr);
ierr = VecScatterCreate(lu->b_seq,is_iden,b,is_iden,&lu->scat_sol);CHKERRQ(ierr);
ierr = ISDestroy(&is_iden);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
}
lu->iparm[IPARM_START_TASK] = API_TASK_ORDERING;
lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]);
} else {
lu->iparm[IPARM_START_TASK] = API_TASK_NUMFACT;
lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]);
}
if (lu->commSize > 1) {
if ((F)->factortype == MAT_FACTOR_LU) {
F_diag = ((Mat_MPIAIJ*)(F)->data)->A;
} else {
F_diag = ((Mat_MPISBAIJ*)(F)->data)->A;
}
F_diag->assembled = PETSC_TRUE;
}
(F)->assembled = PETSC_TRUE;
lu->matstruc = SAME_NONZERO_PATTERN;
lu->CleanUpPastix = PETSC_TRUE;
PetscFunctionReturn(0);
}
PetscErrorCode MatColoringGetDegrees(Mat G,PetscInt distance,PetscInt *degrees)
{
PetscInt j,i,s,e,n,ln,lm,degree,bidx,idx,dist;
Mat lG,*lGs;
IS ris;
PetscErrorCode ierr;
PetscInt *seen;
const PetscInt *gidx;
PetscInt *idxbuf;
PetscInt *distbuf;
PetscInt ncols;
const PetscInt *cols;
PetscBool isSEQAIJ;
Mat_SeqAIJ *aij;
PetscInt *Gi,*Gj;
PetscFunctionBegin;
ierr = MatGetOwnershipRange(G,&s,&e);CHKERRQ(ierr);
n=e-s;
ierr = ISCreateStride(PetscObjectComm((PetscObject)G),n,s,1,&ris);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(G,1,&ris,distance);CHKERRQ(ierr);
ierr = ISSort(ris);CHKERRQ(ierr);
ierr = MatGetSubMatrices(G,1,&ris,&ris,MAT_INITIAL_MATRIX,&lGs);CHKERRQ(ierr);
lG = lGs[0];
ierr = PetscObjectTypeCompare((PetscObject)lG,MATSEQAIJ,&isSEQAIJ);CHKERRQ(ierr);
if (!isSEQAIJ) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_SUP,"Requires an MPI/SEQAIJ Matrix");
ierr = MatGetSize(lG,&ln,&lm);CHKERRQ(ierr);
aij = (Mat_SeqAIJ*)lG->data;
Gi = aij->i;
Gj = aij->j;
ierr = PetscMalloc3(lm,&seen,lm,&idxbuf,lm,&distbuf);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
seen[i]=-1;
}
ierr = ISGetIndices(ris,&gidx);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
if (gidx[i] >= e || gidx[i] < s) continue;
bidx=-1;
ncols = Gi[i+1]-Gi[i];
cols = &(Gj[Gi[i]]);
degree = 0;
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
degree++;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
degrees[gidx[i]-s] = degree;
}
ierr = ISRestoreIndices(ris,&gidx);CHKERRQ(ierr);
ierr = ISDestroy(&ris);CHKERRQ(ierr);
ierr = PetscFree3(seen,idxbuf,distbuf);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lGs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatColoringCreateSmallestLastWeights(MatColoring mc,PetscReal *weights)
{
PetscInt *degrees,*degb,*llprev,*llnext;
PetscInt j,i,s,e,n,nin,ln,lm,degree,maxdegree=0,bidx,idx,dist,distance=mc->dist;
Mat lG,*lGs;
IS ris;
PetscErrorCode ierr;
PetscInt *seen;
const PetscInt *gidx;
PetscInt *idxbuf;
PetscInt *distbuf;
PetscInt ncols,nxt,prv,cur;
const PetscInt *cols;
PetscBool isSEQAIJ;
Mat_SeqAIJ *aij;
PetscInt *Gi,*Gj,*rperm;
Mat G = mc->mat;
PetscReal *lweights,r;
PetscRandom rand;
PetscFunctionBegin;
ierr = MatGetOwnershipRange(G,&s,&e);CHKERRQ(ierr);
n=e-s;
ierr = ISCreateStride(PetscObjectComm((PetscObject)G),n,s,1,&ris);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(G,1,&ris,distance+1);CHKERRQ(ierr);
ierr = ISSort(ris);CHKERRQ(ierr);
ierr = MatGetSubMatrices(G,1,&ris,&ris,MAT_INITIAL_MATRIX,&lGs);CHKERRQ(ierr);
lG = lGs[0];
ierr = PetscObjectTypeCompare((PetscObject)lG,MATSEQAIJ,&isSEQAIJ);CHKERRQ(ierr);
if (!isSEQAIJ) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_ARG_WRONGSTATE,"Requires an MPI/SEQAIJ Matrix");
ierr = MatGetSize(lG,&ln,&lm);CHKERRQ(ierr);
aij = (Mat_SeqAIJ*)lG->data;
Gi = aij->i;
Gj = aij->j;
ierr = PetscMalloc3(lm,&seen,lm,&idxbuf,lm,&distbuf);CHKERRQ(ierr);
ierr = PetscMalloc1(lm,°rees);CHKERRQ(ierr);
ierr = PetscMalloc1(lm,&lweights);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
seen[i]=-1;
lweights[i] = 1.;
}
ierr = ISGetIndices(ris,&gidx);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
bidx=-1;
ncols = Gi[i+1]-Gi[i];
cols = &(Gj[Gi[i]]);
degree = 0;
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
degree++;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
degrees[i] = degree;
if (degree > maxdegree) maxdegree = degree;
}
/* bucket by degree by some random permutation */
ierr = PetscRandomCreate(PetscObjectComm((PetscObject)mc),&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = PetscMalloc1(ln,&rperm);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
ierr = PetscRandomGetValueReal(rand,&r);CHKERRQ(ierr);
lweights[i] = r;
rperm[i]=i;
}
ierr = PetscSortRealWithPermutation(lm,lweights,rperm);CHKERRQ(ierr);
ierr = PetscMalloc1(maxdegree+1,°b);CHKERRQ(ierr);
ierr = PetscMalloc2(ln,&llnext,ln,&llprev);CHKERRQ(ierr);
for (i=0;i<maxdegree+1;i++) {
degb[i] = -1;
}
for (i=0;i<ln;i++) {
llnext[i] = -1;
llprev[i] = -1;
seen[i] = -1;
}
for (i=0;i<ln;i++) {
idx = rperm[i];
llnext[idx] = degb[degrees[idx]];
if (degb[degrees[idx]] > 0) llprev[degb[degrees[idx]]] = idx;
degb[degrees[idx]] = idx;
}
ierr = PetscFree(rperm);CHKERRQ(ierr);
/* remove the lowest degree one */
i=0;
nin=0;
while (i != maxdegree+1) {
for (i=1;i<maxdegree+1; i++) {
if (degb[i] > 0) {
cur = degb[i];
nin++;
degrees[cur] = 0;
degb[i] = llnext[cur];
bidx=-1;
ncols = Gi[cur+1]-Gi[cur];
cols = &(Gj[Gi[cur]]);
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
if (cols[j] != cur) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
nxt=llnext[idx];
prv=llprev[idx];
if (degrees[idx] > 0) {
/* change up the degree of the neighbors still in the graph */
if (lweights[idx] <= lweights[cur]) lweights[idx] = lweights[cur]+1;
if (nxt > 0) {
llprev[nxt] = prv;
}
if (prv > 0) {
llnext[prv] = nxt;
} else {
degb[degrees[idx]] = nxt;
}
degrees[idx]--;
llnext[idx] = degb[degrees[idx]];
llprev[idx] = -1;
if (degb[degrees[idx]] >= 0) {
llprev[degb[degrees[idx]]] = idx;
}
degb[degrees[idx]] = idx;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
}
break;
}
}
}
for (i=0;i<lm;i++) {
if (gidx[i] >= s && gidx[i] < e) {
weights[gidx[i]-s] = lweights[i];
}
}
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFree(degb);CHKERRQ(ierr);
ierr = PetscFree2(llnext,llprev);CHKERRQ(ierr);
ierr = PetscFree(degrees);CHKERRQ(ierr);
ierr = PetscFree(lweights);CHKERRQ(ierr);
ierr = ISRestoreIndices(ris,&gidx);CHKERRQ(ierr);
ierr = ISDestroy(&ris);CHKERRQ(ierr);
ierr = PetscFree3(seen,idxbuf,distbuf);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lGs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}