PetscErrorCode Assemble(MPI_Comm comm,PetscInt n,MatType mtype) { Mat A; PetscInt first,last,i; PetscErrorCode ierr; PetscMPIInt rank,size; PetscFunctionBegin; ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A, PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr); ierr = MatSetType(A,MATMPISBAIJ);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); if (rank < size-1) { ierr = MatMPISBAIJSetPreallocation(A,1,1,PETSC_NULL,1,PETSC_NULL);CHKERRQ(ierr); } else { ierr = MatMPISBAIJSetPreallocation(A,1,2,PETSC_NULL,0,PETSC_NULL);CHKERRQ(ierr); } ierr = MatGetOwnershipRange(A,&first,&last);CHKERRQ(ierr); ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr); last--; for (i=first; i<=last; i++){ ierr = MatSetValue(A,i,i,2.,INSERT_VALUES);CHKERRQ(ierr); if (i != n-1) {ierr = MatSetValue(A,i,n-1,-1.,INSERT_VALUES);CHKERRQ(ierr);} } ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode Assemble(MPI_Comm comm,PetscInt bs,const MatType mtype) { const PetscInt rc[] = {0,1,2,3}; const PetscScalar vals[] = {1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16, 17,18,19,20,21,22,23,24, 25,26,27,28,29,30,31,32, 33,34,35,36,37,38,39,40, 41,42,43,44,45,46,47,48, 49,50,51,52,53,54,55,56, 57,58,49,60,61,62,63,64}; Mat A; PetscViewer viewer; PetscErrorCode ierr; PetscFunctionBegin; ierr = MatCreate(comm,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,4*bs,4*bs);CHKERRQ(ierr); ierr = MatSetType(A,mtype);CHKERRQ(ierr); ierr = MatMPIBAIJSetPreallocation(A,bs,2,NULL,2,NULL);CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(A,bs,2,NULL,2,NULL);CHKERRQ(ierr); ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr); /* All processes contribute a global matrix */ ierr = MatSetValuesBlocked(A,4,rc,4,rc,vals,ADD_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = PetscPrintf(comm,"Matrix %s(%D)\n",mtype,bs);CHKERRQ(ierr); ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr); ierr = PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_INFO_DETAIL);CHKERRQ(ierr); ierr = MatView(A,viewer);CHKERRQ(ierr); ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr); ierr = MatView(A,viewer);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); PetscFunctionReturn(0); }
PETSC_EXTERN PetscErrorCode MatGetFactor_seqsbaij_pastix(Mat A,MatFactorType ftype,Mat *F) { Mat B; PetscErrorCode ierr; Mat_Pastix *pastix; PetscFunctionBegin; if (ftype != MAT_FACTOR_CHOLESKY) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot use PETSc SBAIJ matrices with PaStiX LU, use AIJ matrix"); /* Create the factorization matrix */ ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr); ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);CHKERRQ(ierr); ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr); ierr = MatSeqSBAIJSetPreallocation(B,1,0,NULL);CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(B,1,0,NULL,0,NULL);CHKERRQ(ierr); B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_SBAIJPASTIX; B->ops->view = MatView_PaStiX; ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_pastix);CHKERRQ(ierr); B->factortype = MAT_FACTOR_CHOLESKY; ierr = PetscNewLog(B,&pastix);CHKERRQ(ierr); pastix->CleanUpPastix = PETSC_FALSE; pastix->isAIJ = PETSC_TRUE; pastix->scat_rhs = NULL; pastix->scat_sol = NULL; pastix->Destroy = B->ops->destroy; B->ops->destroy = MatDestroy_Pastix; B->spptr = (void*)pastix; *F = B; PetscFunctionReturn(0); }
int main(int argc, char **args) { Mat A; MatPartitioning part; IS is; PetscInt i,m,N,rstart,rend,nemptyranks,*emptyranks,nbigranks,*bigranks; PetscMPIInt rank,size; PetscErrorCode ierr; ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); nemptyranks = 10; nbigranks = 10; ierr = PetscMalloc2(nemptyranks,PetscInt,&emptyranks,nbigranks,PetscInt,&bigranks);CHKERRQ(ierr); ierr = PetscOptionsBegin(PETSC_COMM_WORLD,PETSC_NULL,"Partitioning example options",PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsIntArray("-emptyranks","Ranks to be skipped by partition","",emptyranks,&nemptyranks,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsIntArray("-bigranks","Ranks to be overloaded","",bigranks,&nbigranks,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsEnd();CHKERRQ(ierr); m = 1; for (i=0; i<nemptyranks; i++) if (rank == emptyranks[i]) m = 0; for (i=0; i<nbigranks; i++) if (rank == bigranks[i]) m = 5; ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,m,m,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A,3,PETSC_NULL);CHKERRQ(ierr); ierr = MatMPIAIJSetPreallocation(A,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr); ierr = MatSeqBAIJSetPreallocation(A,1,3,PETSC_NULL);CHKERRQ(ierr); ierr = MatMPIBAIJSetPreallocation(A,1,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr); ierr = MatSeqSBAIJSetPreallocation(A,1,2,PETSC_NULL);CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(A,1,2,PETSC_NULL,1,PETSC_NULL);CHKERRQ(ierr); ierr = MatGetSize(A,PETSC_NULL,&N);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr); for (i=rstart; i<rend; i++) { const PetscInt cols[] = {(i+N-1)%N,i,(i+1)%N}; const PetscScalar vals[] = {1,1,1}; ierr = MatSetValues(A,1,&i,3,cols,vals,INSERT_VALUES);CHKERRQ(ierr); } ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr); ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr); ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr); ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr); ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = ISDestroy(&is);CHKERRQ(ierr); ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = PetscFree2(emptyranks,bigranks);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
/*@ MatXAIJSetPreallocation - set preallocation for serial and parallel AIJ, BAIJ, and SBAIJ matrices Collective on Mat Input Arguments: + A - matrix being preallocated . bs - block size . dnnz - number of nonzero blocks per block row of diagonal part of parallel matrix . onnz - number of nonzero blocks per block row of off-diagonal part of parallel matrix . dnnzu - number of nonzero blocks per block row of upper-triangular part of diagonal part of parallel matrix - onnzu - number of nonzero blocks per block row of upper-triangular part of off-diagonal part of parallel matrix Level: beginner .seealso: MatSeqAIJSetPreallocation(), MatMPIAIJSetPreallocation(), MatSeqBAIJSetPreallocation(), MatMPIBAIJSetPreallocation(), MatSeqSBAIJSetPreallocation(), MatMPISBAIJSetPreallocation(), PetscSplitOwnership() @*/ PetscErrorCode MatXAIJSetPreallocation(Mat A,PetscInt bs,const PetscInt dnnz[],const PetscInt onnz[],const PetscInt dnnzu[],const PetscInt onnzu[]) { PetscErrorCode ierr; void (*aij)(void); PetscFunctionBegin; ierr = MatSetBlockSize(A,bs); CHKERRQ(ierr); ierr = PetscLayoutSetUp(A->rmap); CHKERRQ(ierr); ierr = PetscLayoutSetUp(A->cmap); CHKERRQ(ierr); ierr = MatSeqBAIJSetPreallocation(A,bs,0,dnnz); CHKERRQ(ierr); ierr = MatMPIBAIJSetPreallocation(A,bs,0,dnnz,0,onnz); CHKERRQ(ierr); ierr = MatSeqSBAIJSetPreallocation(A,bs,0,dnnzu); CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(A,bs,0,dnnzu,0,onnzu); CHKERRQ(ierr); /* In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any good before going on with it. */ ierr = PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij); CHKERRQ(ierr); if (!aij) { ierr = PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij); CHKERRQ(ierr); } if (aij) { if (bs == 1) { ierr = MatSeqAIJSetPreallocation(A,0,dnnz); CHKERRQ(ierr); ierr = MatMPIAIJSetPreallocation(A,0,dnnz,0,onnz); CHKERRQ(ierr); } else { /* Convert block-row precallocation to scalar-row */ PetscInt i,m,*sdnnz,*sonnz; ierr = MatGetLocalSize(A,&m,NULL); CHKERRQ(ierr); ierr = PetscMalloc2((!!dnnz)*m,PetscInt,&sdnnz,(!!onnz)*m,PetscInt,&sonnz); CHKERRQ(ierr); for (i=0; i<m; i++) { if (dnnz) sdnnz[i] = dnnz[i/bs] * bs; if (onnz) sonnz[i] = onnz[i/bs] * bs; } ierr = MatSeqAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL); CHKERRQ(ierr); ierr = MatMPIAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL,0,onnz ? sonnz : NULL); CHKERRQ(ierr); ierr = PetscFree2(sdnnz,sonnz); CHKERRQ(ierr); } } PetscFunctionReturn(0); }
EXTERN_C_BEGIN #undef __FUNCT__ #define __FUNCT__ "MatConvert_MPIAIJ_MPISBAIJ" PetscErrorCode MatConvert_MPIAIJ_MPISBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat) { PetscErrorCode ierr; Mat M; Mat_MPIAIJ *mpimat = (Mat_MPIAIJ*)A->data; Mat_SeqAIJ *Aa = (Mat_SeqAIJ*)mpimat->A->data,*Ba = (Mat_SeqAIJ*)mpimat->B->data; PetscInt *d_nnz,*o_nnz; PetscInt i,j,nz; PetscInt m,n,lm,ln; PetscInt rstart,rend; const PetscScalar *vwork; const PetscInt *cwork; PetscFunctionBegin; if (!A->symmetric) SETERRQ(((PetscObject)A)->comm,PETSC_ERR_USER,"Matrix must be symmetric. Call MatSetOption(mat,MAT_SYMMETRIC,PETSC_TRUE)"); ierr = MatGetSize(A,&m,&n);CHKERRQ(ierr); ierr = MatGetLocalSize(A,&lm,&ln);CHKERRQ(ierr); ierr = PetscMalloc2(lm,PetscInt,&d_nnz,lm,PetscInt,&o_nnz);CHKERRQ(ierr); ierr = MatMarkDiagonal_SeqAIJ(mpimat->A);CHKERRQ(ierr); for (i=0;i<lm;i++){ d_nnz[i] = Aa->i[i+1] - Aa->diag[i]; o_nnz[i] = Ba->i[i+1] - Ba->i[i]; } ierr = MatCreate(((PetscObject)A)->comm,&M);CHKERRQ(ierr); ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr); ierr = MatSetType(M,MATMPISBAIJ);CHKERRQ(ierr); ierr = MatSeqSBAIJSetPreallocation(M,1,0,d_nnz);CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(M,1,0,d_nnz,0,o_nnz);CHKERRQ(ierr); ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr); for (i=rstart;i<rend;i++){ ierr = MatGetRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr); j = 0; while (cwork[j] < i){ j++; nz--;} ierr = MatSetValues(M,1,&i,nz,cwork+j,vwork+j,INSERT_VALUES);CHKERRQ(ierr); ierr = MatRestoreRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr); } ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); if (reuse == MAT_REUSE_MATRIX) { ierr = MatHeaderReplace(A,M);CHKERRQ(ierr); } else { *newmat = M; } PetscFunctionReturn(0); }
PETSC_EXTERN PetscErrorCode MatConvert_MPIBAIJ_MPISBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat) { PetscErrorCode ierr; Mat M; Mat_MPIBAIJ *mpimat = (Mat_MPIBAIJ*)A->data; Mat_SeqBAIJ *Aa = (Mat_SeqBAIJ*)mpimat->A->data,*Ba = (Mat_SeqBAIJ*)mpimat->B->data; PetscInt *d_nnz,*o_nnz; PetscInt i,j,nz; PetscInt m,n,lm,ln; PetscInt rstart,rend; const PetscScalar *vwork; const PetscInt *cwork; PetscInt bs = A->rmap->bs; PetscFunctionBegin; ierr = MatGetSize(A,&m,&n);CHKERRQ(ierr); ierr = MatGetLocalSize(A,&lm,&ln);CHKERRQ(ierr); ierr = PetscMalloc2(lm/bs,&d_nnz,lm/bs,&o_nnz);CHKERRQ(ierr); ierr = MatMarkDiagonal_SeqBAIJ(mpimat->A);CHKERRQ(ierr); for (i=0; i<lm/bs; i++) { d_nnz[i] = Aa->i[i+1] - Aa->diag[i]; o_nnz[i] = Ba->i[i+1] - Ba->i[i]; } ierr = MatCreate(PetscObjectComm((PetscObject)A),&M);CHKERRQ(ierr); ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr); ierr = MatSetType(M,MATMPISBAIJ);CHKERRQ(ierr); ierr = MatSeqSBAIJSetPreallocation(M,bs,0,d_nnz);CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(M,bs,0,d_nnz,0,o_nnz);CHKERRQ(ierr); ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr); ierr = MatSetOption(M,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr); for (i=rstart; i<rend; i++) { ierr = MatGetRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr); j = 0; ierr = MatSetValues(M,1,&i,nz,cwork+j,vwork+j,INSERT_VALUES);CHKERRQ(ierr); ierr = MatRestoreRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr); } ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); if (reuse == MAT_REUSE_MATRIX) { ierr = MatHeaderReplace(A,&M);CHKERRQ(ierr); } else { *newmat = M; } PetscFunctionReturn(0); }
PetscErrorCode MatCreateMPISBSTRM(MPI_Comm comm,PetscInt bs,PetscInt m,PetscInt n,PetscInt M,PetscInt N,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[],Mat *A) { PetscErrorCode ierr; PetscMPIInt size; PetscFunctionBegin; ierr = MatCreate(comm,A);CHKERRQ(ierr); ierr = MatSetSizes(*A,m,n,M,N);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); if (size > 1) { ierr = MatSetType(*A,MATMPISBSTRM);CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(*A,bs,d_nz,d_nnz,o_nz,o_nnz);CHKERRQ(ierr); } else { ierr = MatSetType(*A,MATSEQSBSTRM);CHKERRQ(ierr); ierr = MatSeqSBAIJSetPreallocation(*A,bs,d_nz,d_nnz);CHKERRQ(ierr); } PetscFunctionReturn(0); }
EXTERN_C_END EXTERN_C_BEGIN #undef __FUNCT__ #define __FUNCT__ "MatGetFactor_mpisbaij_spooles" PetscErrorCode MatGetFactor_mpisbaij_spooles(Mat A,MatFactorType ftype,Mat *F) { Mat_Spooles *lu; Mat B; PetscErrorCode ierr; PetscFunctionBegin; /* Create the factorization matrix F */ ierr = MatCreate(((PetscObject)A)->comm,&B);CHKERRQ(ierr); ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);CHKERRQ(ierr); ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(B,1,0,PETSC_NULL,0,PETSC_NULL);CHKERRQ(ierr); ierr = PetscNewLog(B,Mat_Spooles,&lu);CHKERRQ(ierr); B->spptr = lu; lu->flg = DIFFERENT_NONZERO_PATTERN; lu->options.useQR = PETSC_FALSE; if (ftype == MAT_FACTOR_CHOLESKY) { B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_MPISBAIJSpooles; B->ops->view = MatView_Spooles; B->ops->destroy = MatDestroy_MPISBAIJSpooles; ierr = PetscObjectComposeFunctionDynamic((PetscObject)B,"MatFactorGetSolverPackage_C","MatFactorGetSolverPackage_mpisbaij_spooles",MatFactorGetSolverPackage_mpisbaij_spooles);CHKERRQ(ierr); lu->options.symflag = SPOOLES_SYMMETRIC; lu->options.pivotingflag = SPOOLES_NO_PIVOTING; } else SETERRQ(PETSC_ERR_SUP,"Only Cholesky for SBAIJ matrices, use AIJ for LU"); B->factor = ftype; ierr = MPI_Comm_dup(((PetscObject)A)->comm,&(lu->comm_spooles));CHKERRQ(ierr); *F = B; PetscFunctionReturn(0); }
int main(int argc,char **args) { Vec x,y,u,s1,s2; Mat A,sA,sB; PetscRandom rctx; PetscReal r1,r2,rnorm,tol = PETSC_SQRT_MACHINE_EPSILON; PetscScalar one=1.0, neg_one=-1.0, value[3], four=4.0,alpha=0.1; PetscInt n,col[3],n1,block,row,i,j,i2,j2,Ii,J,rstart,rend,bs=1,mbs=16,d_nz=3,o_nz=3,prob=2; PetscErrorCode ierr; PetscMPIInt size,rank; PetscBool flg; MatType type; ierr = PetscInitialize(&argc,&args,(char*)0,help); if (ierr) return ierr; ierr = PetscOptionsGetInt(NULL,NULL,"-mbs",&mbs,NULL); CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL); CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank); CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size); CHKERRQ(ierr); n = mbs*bs; /* Assemble MPISBAIJ matrix sA */ ierr = MatCreate(PETSC_COMM_WORLD,&sA); CHKERRQ(ierr); ierr = MatSetSizes(sA,PETSC_DECIDE,PETSC_DECIDE,n,n); CHKERRQ(ierr); ierr = MatSetType(sA,MATSBAIJ); CHKERRQ(ierr); ierr = MatSetFromOptions(sA); CHKERRQ(ierr); ierr = MatGetType(sA,&type); CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(sA,bs,d_nz,NULL,o_nz,NULL); CHKERRQ(ierr); ierr = MatSeqSBAIJSetPreallocation(sA,bs,d_nz,NULL); CHKERRQ(ierr); ierr = MatSetOption(sA,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE); CHKERRQ(ierr); if (bs == 1) { if (prob == 1) { /* tridiagonal matrix */ value[0] = -1.0; value[1] = 2.0; value[2] = -1.0; for (i=1; i<n-1; i++) { col[0] = i-1; col[1] = i; col[2] = i+1; ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr); } i = n - 1; col[0]=0; col[1] = n - 2; col[2] = n - 1; value[0]= 0.1; value[1]=-1; value[2]=2; ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr); i = 0; col[0] = 0; col[1] = 1; col[2]=n-1; value[0] = 2.0; value[1] = -1.0; value[2]=0.1; ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr); } else if (prob ==2) { /* matrix for the five point stencil */ n1 = (int) PetscSqrtReal((PetscReal)n); if (n1*n1 != n) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"n must be a perfect square of n1"); for (i=0; i<n1; i++) { for (j=0; j<n1; j++) { Ii = j + n1*i; if (i>0) { J = Ii - n1; ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES); CHKERRQ(ierr); } if (i<n1-1) { J = Ii + n1; ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES); CHKERRQ(ierr); } if (j>0) { J = Ii - 1; ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES); CHKERRQ(ierr); } if (j<n1-1) { J = Ii + 1; ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES); CHKERRQ(ierr); } ierr = MatSetValues(sA,1,&Ii,1,&Ii,&four,INSERT_VALUES); CHKERRQ(ierr); } } } /* end of if (bs == 1) */ } else { /* bs > 1 */ for (block=0; block<n/bs; block++) { /* diagonal blocks */ value[0] = -1.0; value[1] = 4.0; value[2] = -1.0; for (i=1+block*bs; i<bs-1+block*bs; i++) { col[0] = i-1; col[1] = i; col[2] = i+1; ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr); } i = bs - 1+block*bs; col[0] = bs - 2+block*bs; col[1] = bs - 1+block*bs; value[0]=-1.0; value[1]=4.0; ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES); CHKERRQ(ierr); i = 0+block*bs; col[0] = 0+block*bs; col[1] = 1+block*bs; value[0]=4.0; value[1] = -1.0; ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES); CHKERRQ(ierr); } /* off-diagonal blocks */ value[0]=-1.0; for (i=0; i<(n/bs-1)*bs; i++) { col[0]=i+bs; ierr = MatSetValues(sA,1,&i,1,col,value,INSERT_VALUES); CHKERRQ(ierr); col[0]=i; row=i+bs; ierr = MatSetValues(sA,1,&row,1,col,value,INSERT_VALUES); CHKERRQ(ierr); } } ierr = MatAssemblyBegin(sA,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); ierr = MatAssemblyEnd(sA,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); /* Test MatView() */ ierr = MatCreateBAIJ(PETSC_COMM_WORLD,bs,PETSC_DECIDE,PETSC_DECIDE,n,n,d_nz,NULL,o_nz,NULL,&A); CHKERRQ(ierr); ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE); CHKERRQ(ierr); if (bs == 1) { if (prob == 1) { /* tridiagonal matrix */ value[0] = -1.0; value[1] = 2.0; value[2] = -1.0; for (i=1; i<n-1; i++) { col[0] = i-1; col[1] = i; col[2] = i+1; ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr); } i = n - 1; col[0]=0; col[1] = n - 2; col[2] = n - 1; value[0]= 0.1; value[1]=-1; value[2]=2; ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr); i = 0; col[0] = 0; col[1] = 1; col[2]=n-1; value[0] = 2.0; value[1] = -1.0; value[2]=0.1; ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr); } else if (prob ==2) { /* matrix for the five point stencil */ n1 = (int) PetscSqrtReal((PetscReal)n); for (i=0; i<n1; i++) { for (j=0; j<n1; j++) { Ii = j + n1*i; if (i>0) { J = Ii - n1; ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES); CHKERRQ(ierr); } if (i<n1-1) { J = Ii + n1; ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES); CHKERRQ(ierr); } if (j>0) { J = Ii - 1; ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES); CHKERRQ(ierr); } if (j<n1-1) { J = Ii + 1; ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES); CHKERRQ(ierr); } ierr = MatSetValues(A,1,&Ii,1,&Ii,&four,INSERT_VALUES); CHKERRQ(ierr); } } } /* end of if (bs == 1) */ } else { /* bs > 1 */ for (block=0; block<n/bs; block++) { /* diagonal blocks */ value[0] = -1.0; value[1] = 4.0; value[2] = -1.0; for (i=1+block*bs; i<bs-1+block*bs; i++) { col[0] = i-1; col[1] = i; col[2] = i+1; ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES); CHKERRQ(ierr); } i = bs - 1+block*bs; col[0] = bs - 2+block*bs; col[1] = bs - 1+block*bs; value[0]=-1.0; value[1]=4.0; ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES); CHKERRQ(ierr); i = 0+block*bs; col[0] = 0+block*bs; col[1] = 1+block*bs; value[0]=4.0; value[1] = -1.0; ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES); CHKERRQ(ierr); } /* off-diagonal blocks */ value[0]=-1.0; for (i=0; i<(n/bs-1)*bs; i++) { col[0]=i+bs; ierr = MatSetValues(A,1,&i,1,col,value,INSERT_VALUES); CHKERRQ(ierr); col[0]=i; row=i+bs; ierr = MatSetValues(A,1,&row,1,col,value,INSERT_VALUES); CHKERRQ(ierr); } } ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); /* Test MatGetSize(), MatGetLocalSize() */ ierr = MatGetSize(sA, &i,&j); CHKERRQ(ierr); ierr = MatGetSize(A, &i2,&j2); CHKERRQ(ierr); i -= i2; j -= j2; if (i || j) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatGetSize()\n",rank); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } ierr = MatGetLocalSize(sA, &i,&j); CHKERRQ(ierr); ierr = MatGetLocalSize(A, &i2,&j2); CHKERRQ(ierr); i2 -= i; j2 -= j; if (i2 || j2) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatGetLocalSize()\n",rank); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } /* vectors */ /*--------------------*/ /* i is obtained from MatGetLocalSize() */ ierr = VecCreate(PETSC_COMM_WORLD,&x); CHKERRQ(ierr); ierr = VecSetSizes(x,i,PETSC_DECIDE); CHKERRQ(ierr); ierr = VecSetFromOptions(x); CHKERRQ(ierr); ierr = VecDuplicate(x,&y); CHKERRQ(ierr); ierr = VecDuplicate(x,&u); CHKERRQ(ierr); ierr = VecDuplicate(x,&s1); CHKERRQ(ierr); ierr = VecDuplicate(x,&s2); CHKERRQ(ierr); ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx); CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rctx); CHKERRQ(ierr); ierr = VecSetRandom(x,rctx); CHKERRQ(ierr); ierr = VecSet(u,one); CHKERRQ(ierr); /* Test MatNorm() */ ierr = MatNorm(A,NORM_FROBENIUS,&r1); CHKERRQ(ierr); ierr = MatNorm(sA,NORM_FROBENIUS,&r2); CHKERRQ(ierr); rnorm = PetscAbsReal(r1-r2)/r2; if (rnorm > tol && !rank) { ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_FROBENIUS(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs); CHKERRQ(ierr); } ierr = MatNorm(A,NORM_INFINITY,&r1); CHKERRQ(ierr); ierr = MatNorm(sA,NORM_INFINITY,&r2); CHKERRQ(ierr); rnorm = PetscAbsReal(r1-r2)/r2; if (rnorm > tol && !rank) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: MatNorm_INFINITY(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs); CHKERRQ(ierr); } ierr = MatNorm(A,NORM_1,&r1); CHKERRQ(ierr); ierr = MatNorm(sA,NORM_1,&r2); CHKERRQ(ierr); rnorm = PetscAbsReal(r1-r2)/r2; if (rnorm > tol && !rank) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: MatNorm_1(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs); CHKERRQ(ierr); } /* Test MatGetOwnershipRange() */ ierr = MatGetOwnershipRange(sA,&rstart,&rend); CHKERRQ(ierr); ierr = MatGetOwnershipRange(A,&i2,&j2); CHKERRQ(ierr); i2 -= rstart; j2 -= rend; if (i2 || j2) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MaGetOwnershipRange()\n",rank); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } /* Test MatDiagonalScale() */ ierr = MatDiagonalScale(A,x,x); CHKERRQ(ierr); ierr = MatDiagonalScale(sA,x,x); CHKERRQ(ierr); ierr = MatMultEqual(A,sA,10,&flg); CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDiagonalScale"); /* Test MatGetDiagonal(), MatScale() */ ierr = MatGetDiagonal(A,s1); CHKERRQ(ierr); ierr = MatGetDiagonal(sA,s2); CHKERRQ(ierr); ierr = VecNorm(s1,NORM_1,&r1); CHKERRQ(ierr); ierr = VecNorm(s2,NORM_1,&r2); CHKERRQ(ierr); r1 -= r2; if (r1<-tol || r1>tol) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDiagonalScale() or MatGetDiagonal(), r1=%g \n",rank,(double)r1); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } ierr = MatScale(A,alpha); CHKERRQ(ierr); ierr = MatScale(sA,alpha); CHKERRQ(ierr); /* Test MatGetRowMaxAbs() */ ierr = MatGetRowMaxAbs(A,s1,NULL); CHKERRQ(ierr); ierr = MatGetRowMaxAbs(sA,s2,NULL); CHKERRQ(ierr); ierr = VecNorm(s1,NORM_1,&r1); CHKERRQ(ierr); ierr = VecNorm(s2,NORM_1,&r2); CHKERRQ(ierr); r1 -= r2; if (r1<-tol || r1>tol) { ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetRowMaxAbs() \n"); CHKERRQ(ierr); } /* Test MatMult(), MatMultAdd() */ ierr = MatMultEqual(A,sA,10,&flg); CHKERRQ(ierr); if (!flg) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMult() or MatScale()\n",rank); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } ierr = MatMultAddEqual(A,sA,10,&flg); CHKERRQ(ierr); if (!flg) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMultAdd()\n",rank); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } /* Test MatMultTranspose(), MatMultTransposeAdd() */ for (i=0; i<10; i++) { ierr = VecSetRandom(x,rctx); CHKERRQ(ierr); ierr = MatMultTranspose(A,x,s1); CHKERRQ(ierr); ierr = MatMultTranspose(sA,x,s2); CHKERRQ(ierr); ierr = VecNorm(s1,NORM_1,&r1); CHKERRQ(ierr); ierr = VecNorm(s2,NORM_1,&r2); CHKERRQ(ierr); r1 -= r2; if (r1<-tol || r1>tol) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMult() or MatScale(), err=%g\n",rank,(double)r1); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } } for (i=0; i<10; i++) { ierr = VecSetRandom(x,rctx); CHKERRQ(ierr); ierr = VecSetRandom(y,rctx); CHKERRQ(ierr); ierr = MatMultTransposeAdd(A,x,y,s1); CHKERRQ(ierr); ierr = MatMultTransposeAdd(sA,x,y,s2); CHKERRQ(ierr); ierr = VecNorm(s1,NORM_1,&r1); CHKERRQ(ierr); ierr = VecNorm(s2,NORM_1,&r2); CHKERRQ(ierr); r1 -= r2; if (r1<-tol || r1>tol) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMultAdd(), err=%g \n",rank,(double)r1); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } } /* Test MatDuplicate() */ ierr = MatDuplicate(sA,MAT_COPY_VALUES,&sB); CHKERRQ(ierr); ierr = MatEqual(sA,sB,&flg); CHKERRQ(ierr); if (!flg) { ierr = PetscPrintf(PETSC_COMM_WORLD," Error in MatDuplicate(), sA != sB \n"); CHKERRQ(ierr); CHKERRQ(ierr); } ierr = MatMultEqual(sA,sB,5,&flg); CHKERRQ(ierr); if (!flg) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDuplicate() or MatMult()\n",rank); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } ierr = MatMultAddEqual(sA,sB,5,&flg); CHKERRQ(ierr); if (!flg) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDuplicate() or MatMultAdd(()\n",rank); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); } ierr = MatDestroy(&sB); CHKERRQ(ierr); ierr = VecDestroy(&u); CHKERRQ(ierr); ierr = VecDestroy(&x); CHKERRQ(ierr); ierr = VecDestroy(&y); CHKERRQ(ierr); ierr = VecDestroy(&s1); CHKERRQ(ierr); ierr = VecDestroy(&s2); CHKERRQ(ierr); ierr = MatDestroy(&sA); CHKERRQ(ierr); ierr = MatDestroy(&A); CHKERRQ(ierr); ierr = PetscRandomDestroy(&rctx); CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
PETSC_EXTERN PetscErrorCode MatISSetMPIXAIJPreallocation_Private(Mat A, Mat B, PetscBool maxreduce) { Mat_IS *matis = (Mat_IS*)(A->data); PetscInt *my_dnz,*my_onz,*dnz,*onz,*mat_ranges,*row_ownership; const PetscInt *global_indices_r,*global_indices_c; PetscInt i,j,bs,rows,cols; PetscInt lrows,lcols; PetscInt local_rows,local_cols; PetscMPIInt nsubdomains; PetscBool isdense,issbaij; PetscErrorCode ierr; PetscFunctionBegin; ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&nsubdomains);CHKERRQ(ierr); ierr = MatGetSize(A,&rows,&cols);CHKERRQ(ierr); ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr); ierr = MatGetSize(matis->A,&local_rows,&local_cols);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr); ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr); if (A->rmap->mapping != A->cmap->mapping) { ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr); } else { global_indices_c = global_indices_r; } if (issbaij) { ierr = MatGetRowUpperTriangular(matis->A);CHKERRQ(ierr); } /* An SF reduce is needed to sum up properly on shared rows. Note that generally preallocation is not exact, since it overestimates nonzeros */ if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */ ierr = MatISComputeSF_Private(A);CHKERRQ(ierr); } ierr = MatGetLocalSize(A,&lrows,&lcols);CHKERRQ(ierr); ierr = MatPreallocateInitialize(PetscObjectComm((PetscObject)A),lrows,lcols,dnz,onz);CHKERRQ(ierr); /* All processes need to compute entire row ownership */ ierr = PetscMalloc1(rows,&row_ownership);CHKERRQ(ierr); ierr = MatGetOwnershipRanges(A,(const PetscInt**)&mat_ranges);CHKERRQ(ierr); for (i=0;i<nsubdomains;i++) { for (j=mat_ranges[i];j<mat_ranges[i+1];j++) { row_ownership[j] = i; } } /* my_dnz and my_onz contains exact contribution to preallocation from each local mat then, they will be summed up properly. This way, preallocation is always sufficient */ ierr = PetscCalloc2(local_rows,&my_dnz,local_rows,&my_onz);CHKERRQ(ierr); /* preallocation as a MATAIJ */ if (isdense) { /* special case for dense local matrices */ for (i=0;i<local_rows;i++) { PetscInt index_row = global_indices_r[i]; for (j=i;j<local_rows;j++) { PetscInt owner = row_ownership[index_row]; PetscInt index_col = global_indices_c[j]; if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */ my_dnz[i] += 1; } else { /* offdiag block */ my_onz[i] += 1; } /* same as before, interchanging rows and cols */ if (i != j) { owner = row_ownership[index_col]; if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) { my_dnz[j] += 1; } else { my_onz[j] += 1; } } } } } else { /* TODO: this could be optimized using MatGetRowIJ */ for (i=0;i<local_rows;i++) { const PetscInt *cols; PetscInt ncols,index_row = global_indices_r[i]; ierr = MatGetRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr); for (j=0;j<ncols;j++) { PetscInt owner = row_ownership[index_row]; PetscInt index_col = global_indices_c[cols[j]]; if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */ my_dnz[i] += 1; } else { /* offdiag block */ my_onz[i] += 1; } /* same as before, interchanging rows and cols */ if (issbaij && index_col != index_row) { owner = row_ownership[index_col]; if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) { my_dnz[cols[j]] += 1; } else { my_onz[cols[j]] += 1; } } } ierr = MatRestoreRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr); } } ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr); if (global_indices_c != global_indices_r) { ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr); } ierr = PetscFree(row_ownership);CHKERRQ(ierr); /* Reduce my_dnz and my_onz */ if (maxreduce) { ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr); ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr); ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr); ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr); } else { ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr); ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr); ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr); ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr); } ierr = PetscFree2(my_dnz,my_onz);CHKERRQ(ierr); /* Resize preallocation if overestimated */ for (i=0;i<lrows;i++) { dnz[i] = PetscMin(dnz[i],lcols); onz[i] = PetscMin(onz[i],cols-lcols); } /* set preallocation */ ierr = MatMPIAIJSetPreallocation(B,0,dnz,0,onz);CHKERRQ(ierr); for (i=0;i<lrows/bs;i++) { dnz[i] = dnz[i*bs]/bs; onz[i] = onz[i*bs]/bs; } ierr = MatMPIBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr); ierr = MatMPISBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr); ierr = MatPreallocateFinalize(dnz,onz);CHKERRQ(ierr); if (issbaij) { ierr = MatRestoreRowUpperTriangular(matis->A);CHKERRQ(ierr); } PetscFunctionReturn(0); }
PetscErrorCode MatISGetMPIXAIJ_IS(Mat mat, MatReuse reuse, Mat *M) { Mat_IS *matis = (Mat_IS*)(mat->data); /* info on mat */ /* ISLocalToGlobalMapping rmapping,cmapping; */ PetscInt bs,rows,cols; PetscInt lrows,lcols; PetscInt local_rows,local_cols; PetscBool isdense,issbaij,issbaij_red; /* values insertion */ PetscScalar *array; PetscInt *local_indices,*global_indices; /* work */ PetscInt i,j,index_row; PetscErrorCode ierr; PetscFunctionBegin; /* MISSING CHECKS - rectangular case not covered (it is not allowed by MATIS) */ /* get info from mat */ /* ierr = MatGetLocalToGlobalMapping(mat,&rmapping,&cmapping);CHKERRQ(ierr); */ ierr = MatGetSize(mat,&rows,&cols);CHKERRQ(ierr); ierr = MatGetBlockSize(mat,&bs);CHKERRQ(ierr); ierr = MatGetSize(matis->A,&local_rows,&local_cols);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr); /* work */ ierr = PetscMalloc1(local_rows,&local_indices);CHKERRQ(ierr); for (i=0;i<local_rows;i++) local_indices[i]=i; /* map indices of local mat to global values */ ierr = PetscMalloc(PetscMax(local_cols,local_rows)*sizeof(*global_indices),&global_indices);CHKERRQ(ierr); /* ierr = ISLocalToGlobalMappingApply(rmapping,local_rows,local_indices,global_indices);CHKERRQ(ierr); */ ierr = ISLocalToGlobalMappingApply(matis->mapping,local_rows,local_indices,global_indices);CHKERRQ(ierr); if (issbaij) { ierr = MatGetRowUpperTriangular(matis->A);CHKERRQ(ierr); } if (reuse == MAT_INITIAL_MATRIX) { Mat new_mat; MatType new_mat_type; Vec vec_dnz,vec_onz; PetscScalar *my_dnz,*my_onz; PetscInt *dnz,*onz,*mat_ranges,*row_ownership; PetscInt index_col,owner; PetscMPIInt nsubdomains; /* determining new matrix type */ ierr = MPI_Allreduce(&issbaij,&issbaij_red,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)mat));CHKERRQ(ierr); if (issbaij_red) { new_mat_type = MATSBAIJ; } else { if (bs>1) { new_mat_type = MATBAIJ; } else { new_mat_type = MATAIJ; } } ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&nsubdomains);CHKERRQ(ierr); ierr = MatCreate(PetscObjectComm((PetscObject)mat),&new_mat);CHKERRQ(ierr); ierr = MatSetSizes(new_mat,PETSC_DECIDE,PETSC_DECIDE,rows,cols);CHKERRQ(ierr); ierr = MatSetBlockSize(new_mat,bs);CHKERRQ(ierr); ierr = MatSetType(new_mat,new_mat_type);CHKERRQ(ierr); ierr = MatSetUp(new_mat);CHKERRQ(ierr); ierr = MatGetLocalSize(new_mat,&lrows,&lcols);CHKERRQ(ierr); /* preallocation */ ierr = MatPreallocateInitialize(PetscObjectComm((PetscObject)new_mat),lrows,lcols,dnz,onz);CHKERRQ(ierr); /* Some vectors are needed to sum up properly on shared interface dofs. Preallocation macros cannot do the job. Note that preallocation is not exact, since it overestimates nonzeros */ ierr = MatCreateVecs(new_mat,NULL,&vec_dnz);CHKERRQ(ierr); /* ierr = VecSetLocalToGlobalMapping(vec_dnz,rmapping);CHKERRQ(ierr); */ ierr = VecSetLocalToGlobalMapping(vec_dnz,matis->mapping);CHKERRQ(ierr); ierr = VecDuplicate(vec_dnz,&vec_onz);CHKERRQ(ierr); /* All processes need to compute entire row ownership */ ierr = PetscMalloc1(rows,&row_ownership);CHKERRQ(ierr); ierr = MatGetOwnershipRanges(new_mat,(const PetscInt**)&mat_ranges);CHKERRQ(ierr); for (i=0;i<nsubdomains;i++) { for (j=mat_ranges[i];j<mat_ranges[i+1];j++) { row_ownership[j]=i; } } /* my_dnz and my_onz contains exact contribution to preallocation from each local mat then, they will be summed up properly. This way, preallocation is always sufficient */ ierr = PetscMalloc1(local_rows,&my_dnz);CHKERRQ(ierr); ierr = PetscMalloc1(local_rows,&my_onz);CHKERRQ(ierr); ierr = PetscMemzero(my_dnz,local_rows*sizeof(*my_dnz));CHKERRQ(ierr); ierr = PetscMemzero(my_onz,local_rows*sizeof(*my_onz));CHKERRQ(ierr); /* preallocation as a MATAIJ */ if (isdense) { /* special case for dense local matrices */ for (i=0;i<local_rows;i++) { index_row = global_indices[i]; for (j=i;j<local_rows;j++) { owner = row_ownership[index_row]; index_col = global_indices[j]; if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */ my_dnz[i] += 1.0; } else { /* offdiag block */ my_onz[i] += 1.0; } /* same as before, interchanging rows and cols */ if (i != j) { owner = row_ownership[index_col]; if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) { my_dnz[j] += 1.0; } else { my_onz[j] += 1.0; } } } } } else { for (i=0;i<local_rows;i++) { PetscInt ncols; const PetscInt *cols; index_row = global_indices[i]; ierr = MatGetRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr); for (j=0;j<ncols;j++) { owner = row_ownership[index_row]; index_col = global_indices[cols[j]]; if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */ my_dnz[i] += 1.0; } else { /* offdiag block */ my_onz[i] += 1.0; } /* same as before, interchanging rows and cols */ if (issbaij) { owner = row_ownership[index_col]; if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) { my_dnz[j] += 1.0; } else { my_onz[j] += 1.0; } } } ierr = MatRestoreRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr); } } ierr = VecSet(vec_dnz,0.0);CHKERRQ(ierr); ierr = VecSet(vec_onz,0.0);CHKERRQ(ierr); if (local_rows) { /* multilevel guard */ ierr = VecSetValuesLocal(vec_dnz,local_rows,local_indices,my_dnz,ADD_VALUES);CHKERRQ(ierr); ierr = VecSetValuesLocal(vec_onz,local_rows,local_indices,my_onz,ADD_VALUES);CHKERRQ(ierr); } ierr = VecAssemblyBegin(vec_dnz);CHKERRQ(ierr); ierr = VecAssemblyBegin(vec_onz);CHKERRQ(ierr); ierr = VecAssemblyEnd(vec_dnz);CHKERRQ(ierr); ierr = VecAssemblyEnd(vec_onz);CHKERRQ(ierr); ierr = PetscFree(my_dnz);CHKERRQ(ierr); ierr = PetscFree(my_onz);CHKERRQ(ierr); ierr = PetscFree(row_ownership);CHKERRQ(ierr); /* set computed preallocation in dnz and onz */ ierr = VecGetArray(vec_dnz,&array);CHKERRQ(ierr); for (i=0; i<lrows; i++) dnz[i] = (PetscInt)PetscRealPart(array[i]); ierr = VecRestoreArray(vec_dnz,&array);CHKERRQ(ierr); ierr = VecGetArray(vec_onz,&array);CHKERRQ(ierr); for (i=0;i<lrows;i++) onz[i] = (PetscInt)PetscRealPart(array[i]); ierr = VecRestoreArray(vec_onz,&array);CHKERRQ(ierr); ierr = VecDestroy(&vec_dnz);CHKERRQ(ierr); ierr = VecDestroy(&vec_onz);CHKERRQ(ierr); /* Resize preallocation if overestimated */ for (i=0;i<lrows;i++) { dnz[i] = PetscMin(dnz[i],lcols); onz[i] = PetscMin(onz[i],cols-lcols); } /* set preallocation */ ierr = MatMPIAIJSetPreallocation(new_mat,0,dnz,0,onz);CHKERRQ(ierr); for (i=0;i<lrows/bs;i++) { dnz[i] = dnz[i*bs]/bs; onz[i] = onz[i*bs]/bs; } ierr = MatMPIBAIJSetPreallocation(new_mat,bs,0,dnz,0,onz);CHKERRQ(ierr); for (i=0;i<lrows/bs;i++) { dnz[i] = dnz[i]-i; } ierr = MatMPISBAIJSetPreallocation(new_mat,bs,0,dnz,0,onz);CHKERRQ(ierr); ierr = MatPreallocateFinalize(dnz,onz);CHKERRQ(ierr); *M = new_mat; } else { PetscInt mbs,mrows,mcols; /* some checks */ ierr = MatGetBlockSize(*M,&mbs);CHKERRQ(ierr); ierr = MatGetSize(*M,&mrows,&mcols);CHKERRQ(ierr); if (mrows != rows) { SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of rows (%d != %d)",rows,mrows); } if (mrows != rows) { SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of cols (%d != %d)",cols,mcols); } if (mbs != bs) { SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong block size (%d != %d)",bs,mbs); } ierr = MatZeroEntries(*M);CHKERRQ(ierr); } /* set local to global mappings */ /* ierr = MatSetLocalToGlobalMapping(*M,rmapping,cmapping);CHKERRQ(ierr); */ /* Set values */ if (isdense) { /* special case for dense local matrices */ ierr = MatSetOption(*M,MAT_ROW_ORIENTED,PETSC_FALSE);CHKERRQ(ierr); ierr = MatDenseGetArray(matis->A,&array);CHKERRQ(ierr); ierr = MatSetValues(*M,local_rows,global_indices,local_cols,global_indices,array,ADD_VALUES);CHKERRQ(ierr); ierr = MatDenseRestoreArray(matis->A,&array);CHKERRQ(ierr); ierr = PetscFree(local_indices);CHKERRQ(ierr); ierr = PetscFree(global_indices);CHKERRQ(ierr); } else { /* very basic values insertion for all other matrix types */ ierr = PetscFree(local_indices);CHKERRQ(ierr); for (i=0;i<local_rows;i++) { ierr = MatGetRow(matis->A,i,&j,(const PetscInt**)&local_indices,(const PetscScalar**)&array);CHKERRQ(ierr); /* ierr = MatSetValuesLocal(*M,1,&i,j,local_indices,array,ADD_VALUES);CHKERRQ(ierr); */ ierr = ISLocalToGlobalMappingApply(matis->mapping,j,local_indices,global_indices);CHKERRQ(ierr); ierr = ISLocalToGlobalMappingApply(matis->mapping,1,&i,&index_row);CHKERRQ(ierr); ierr = MatSetValues(*M,1,&index_row,j,global_indices,array,ADD_VALUES);CHKERRQ(ierr); ierr = MatRestoreRow(matis->A,i,&j,(const PetscInt**)&local_indices,(const PetscScalar**)&array);CHKERRQ(ierr); } ierr = PetscFree(global_indices);CHKERRQ(ierr); } ierr = MatAssemblyBegin(*M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(*M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); if (isdense) { ierr = MatSetOption(*M,MAT_ROW_ORIENTED,PETSC_TRUE);CHKERRQ(ierr); } if (issbaij) { ierr = MatRestoreRowUpperTriangular(matis->A);CHKERRQ(ierr); } PetscFunctionReturn(0); }