PetscErrorCode Assemble(MPI_Comm comm,PetscInt n,MatType mtype)
{
Mat A;
PetscInt first,last,i;
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscFunctionBegin;
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A, PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetType(A,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
if (rank < size-1) {
ierr = MatMPISBAIJSetPreallocation(A,1,1,PETSC_NULL,1,PETSC_NULL);CHKERRQ(ierr);
} else {
ierr = MatMPISBAIJSetPreallocation(A,1,2,PETSC_NULL,0,PETSC_NULL);CHKERRQ(ierr);
}
ierr = MatGetOwnershipRange(A,&first,&last);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
last--;
for (i=first; i<=last; i++){
ierr = MatSetValue(A,i,i,2.,INSERT_VALUES);CHKERRQ(ierr);
if (i != n-1) {ierr = MatSetValue(A,i,n-1,-1.,INSERT_VALUES);CHKERRQ(ierr);}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode Assemble(MPI_Comm comm,PetscInt bs,const MatType mtype)
{
const PetscInt rc[] = {0,1,2,3};
const PetscScalar vals[] = {1, 2, 3, 4, 5, 6, 7, 8,
9,10,11,12,13,14,15,16,
17,18,19,20,21,22,23,24,
25,26,27,28,29,30,31,32,
33,34,35,36,37,38,39,40,
41,42,43,44,45,46,47,48,
49,50,51,52,53,54,55,56,
57,58,49,60,61,62,63,64};
Mat A;
PetscViewer viewer;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MatCreate(comm,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,4*bs,4*bs);CHKERRQ(ierr);
ierr = MatSetType(A,mtype);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,2,NULL,2,NULL);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,bs,2,NULL,2,NULL);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
/* All processes contribute a global matrix */
ierr = MatSetValuesBlocked(A,4,rc,4,rc,vals,ADD_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Matrix %s(%D)\n",mtype,bs);CHKERRQ(ierr);
ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_INFO_DETAIL);CHKERRQ(ierr);
ierr = MatView(A,viewer);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
ierr = MatView(A,viewer);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode MatGetFactor_seqsbaij_pastix(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
PetscErrorCode ierr;
Mat_Pastix *pastix;
PetscFunctionBegin;
if (ftype != MAT_FACTOR_CHOLESKY) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot use PETSc SBAIJ matrices with PaStiX LU, use AIJ matrix");
/* Create the factorization matrix */
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(B,1,0,NULL);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(B,1,0,NULL,0,NULL);CHKERRQ(ierr);
B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_SBAIJPASTIX;
B->ops->view = MatView_PaStiX;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_pastix);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_CHOLESKY;
ierr = PetscNewLog(B,&pastix);CHKERRQ(ierr);
pastix->CleanUpPastix = PETSC_FALSE;
pastix->isAIJ = PETSC_TRUE;
pastix->scat_rhs = NULL;
pastix->scat_sol = NULL;
pastix->Destroy = B->ops->destroy;
B->ops->destroy = MatDestroy_Pastix;
B->spptr = (void*)pastix;
*F = B;
PetscFunctionReturn(0);
}
int main(int argc, char **args)
{
Mat A;
MatPartitioning part;
IS is;
PetscInt i,m,N,rstart,rend,nemptyranks,*emptyranks,nbigranks,*bigranks;
PetscMPIInt rank,size;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
nemptyranks = 10;
nbigranks = 10;
ierr = PetscMalloc2(nemptyranks,PetscInt,&emptyranks,nbigranks,PetscInt,&bigranks);CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,PETSC_NULL,"Partitioning example options",PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsIntArray("-emptyranks","Ranks to be skipped by partition","",emptyranks,&nemptyranks,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsIntArray("-bigranks","Ranks to be overloaded","",bigranks,&nbigranks,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
m = 1;
for (i=0; i<nemptyranks; i++) if (rank == emptyranks[i]) m = 0;
for (i=0; i<nbigranks; i++) if (rank == bigranks[i]) m = 5;
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m,m,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,1,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,1,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,1,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,1,2,PETSC_NULL,1,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetSize(A,PETSC_NULL,&N);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
const PetscInt cols[] = {(i+N-1)%N,i,(i+1)%N};
const PetscScalar vals[] = {1,1,1};
ierr = MatSetValues(A,1,&i,3,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFree2(emptyranks,bigranks);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@
MatXAIJSetPreallocation - set preallocation for serial and parallel AIJ, BAIJ, and SBAIJ matrices
Collective on Mat
Input Arguments:
+ A - matrix being preallocated
. bs - block size
. dnnz - number of nonzero blocks per block row of diagonal part of parallel matrix
. onnz - number of nonzero blocks per block row of off-diagonal part of parallel matrix
. dnnzu - number of nonzero blocks per block row of upper-triangular part of diagonal part of parallel matrix
- onnzu - number of nonzero blocks per block row of upper-triangular part of off-diagonal part of parallel matrix
Level: beginner
.seealso: MatSeqAIJSetPreallocation(), MatMPIAIJSetPreallocation(), MatSeqBAIJSetPreallocation(), MatMPIBAIJSetPreallocation(), MatSeqSBAIJSetPreallocation(), MatMPISBAIJSetPreallocation(),
PetscSplitOwnership()
@*/
PetscErrorCode MatXAIJSetPreallocation(Mat A,PetscInt bs,const PetscInt dnnz[],const PetscInt onnz[],const PetscInt dnnzu[],const PetscInt onnzu[])
{
PetscErrorCode ierr;
void (*aij)(void);
PetscFunctionBegin;
ierr = MatSetBlockSize(A,bs);
CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->rmap);
CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->cmap);
CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,0,dnnz);
CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,0,dnnz,0,onnz);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,bs,0,dnnzu);
CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,bs,0,dnnzu,0,onnzu);
CHKERRQ(ierr);
/*
In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
good before going on with it.
*/
ierr = PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
CHKERRQ(ierr);
if (!aij) {
ierr = PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
CHKERRQ(ierr);
}
if (aij) {
if (bs == 1) {
ierr = MatSeqAIJSetPreallocation(A,0,dnnz);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz,0,onnz);
CHKERRQ(ierr);
} else { /* Convert block-row precallocation to scalar-row */
PetscInt i,m,*sdnnz,*sonnz;
ierr = MatGetLocalSize(A,&m,NULL);
CHKERRQ(ierr);
ierr = PetscMalloc2((!!dnnz)*m,PetscInt,&sdnnz,(!!onnz)*m,PetscInt,&sonnz);
CHKERRQ(ierr);
for (i=0; i<m; i++) {
if (dnnz) sdnnz[i] = dnnz[i/bs] * bs;
if (onnz) sonnz[i] = onnz[i/bs] * bs;
}
ierr = MatSeqAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL,0,onnz ? sonnz : NULL);
CHKERRQ(ierr);
ierr = PetscFree2(sdnnz,sonnz);
CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "MatConvert_MPIAIJ_MPISBAIJ"
PetscErrorCode MatConvert_MPIAIJ_MPISBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat)
{
PetscErrorCode ierr;
Mat M;
Mat_MPIAIJ *mpimat = (Mat_MPIAIJ*)A->data;
Mat_SeqAIJ *Aa = (Mat_SeqAIJ*)mpimat->A->data,*Ba = (Mat_SeqAIJ*)mpimat->B->data;
PetscInt *d_nnz,*o_nnz;
PetscInt i,j,nz;
PetscInt m,n,lm,ln;
PetscInt rstart,rend;
const PetscScalar *vwork;
const PetscInt *cwork;
PetscFunctionBegin;
if (!A->symmetric) SETERRQ(((PetscObject)A)->comm,PETSC_ERR_USER,"Matrix must be symmetric. Call MatSetOption(mat,MAT_SYMMETRIC,PETSC_TRUE)");
ierr = MatGetSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&lm,&ln);CHKERRQ(ierr);
ierr = PetscMalloc2(lm,PetscInt,&d_nnz,lm,PetscInt,&o_nnz);CHKERRQ(ierr);
ierr = MatMarkDiagonal_SeqAIJ(mpimat->A);CHKERRQ(ierr);
for (i=0;i<lm;i++){
d_nnz[i] = Aa->i[i+1] - Aa->diag[i];
o_nnz[i] = Ba->i[i+1] - Ba->i[i];
}
ierr = MatCreate(((PetscObject)A)->comm,&M);CHKERRQ(ierr);
ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr);
ierr = MatSetType(M,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(M,1,0,d_nnz);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(M,1,0,d_nnz,0,o_nnz);CHKERRQ(ierr);
ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart;i<rend;i++){
ierr = MatGetRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
j = 0;
while (cwork[j] < i){ j++; nz--;}
ierr = MatSetValues(M,1,&i,nz,cwork+j,vwork+j,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_REUSE_MATRIX) {
ierr = MatHeaderReplace(A,M);CHKERRQ(ierr);
} else {
*newmat = M;
}
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode MatConvert_MPIBAIJ_MPISBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat)
{
PetscErrorCode ierr;
Mat M;
Mat_MPIBAIJ *mpimat = (Mat_MPIBAIJ*)A->data;
Mat_SeqBAIJ *Aa = (Mat_SeqBAIJ*)mpimat->A->data,*Ba = (Mat_SeqBAIJ*)mpimat->B->data;
PetscInt *d_nnz,*o_nnz;
PetscInt i,j,nz;
PetscInt m,n,lm,ln;
PetscInt rstart,rend;
const PetscScalar *vwork;
const PetscInt *cwork;
PetscInt bs = A->rmap->bs;
PetscFunctionBegin;
ierr = MatGetSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&lm,&ln);CHKERRQ(ierr);
ierr = PetscMalloc2(lm/bs,&d_nnz,lm/bs,&o_nnz);CHKERRQ(ierr);
ierr = MatMarkDiagonal_SeqBAIJ(mpimat->A);CHKERRQ(ierr);
for (i=0; i<lm/bs; i++) {
d_nnz[i] = Aa->i[i+1] - Aa->diag[i];
o_nnz[i] = Ba->i[i+1] - Ba->i[i];
}
ierr = MatCreate(PetscObjectComm((PetscObject)A),&M);CHKERRQ(ierr);
ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr);
ierr = MatSetType(M,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(M,bs,0,d_nnz);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(M,bs,0,d_nnz,0,o_nnz);CHKERRQ(ierr);
ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatSetOption(M,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatGetRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
j = 0;
ierr = MatSetValues(M,1,&i,nz,cwork+j,vwork+j,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_REUSE_MATRIX) {
ierr = MatHeaderReplace(A,&M);CHKERRQ(ierr);
} else {
*newmat = M;
}
PetscFunctionReturn(0);
}
PetscErrorCode MatCreateMPISBSTRM(MPI_Comm comm,PetscInt bs,PetscInt m,PetscInt n,PetscInt M,PetscInt N,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[],Mat *A)
{
PetscErrorCode ierr;
PetscMPIInt size;
PetscFunctionBegin;
ierr = MatCreate(comm,A);CHKERRQ(ierr);
ierr = MatSetSizes(*A,m,n,M,N);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size > 1) {
ierr = MatSetType(*A,MATMPISBSTRM);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(*A,bs,d_nz,d_nnz,o_nz,o_nnz);CHKERRQ(ierr);
} else {
ierr = MatSetType(*A,MATSEQSBSTRM);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(*A,bs,d_nz,d_nnz);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
EXTERN_C_END
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "MatGetFactor_mpisbaij_spooles"
PetscErrorCode MatGetFactor_mpisbaij_spooles(Mat A,MatFactorType ftype,Mat *F)
{
Mat_Spooles *lu;
Mat B;
PetscErrorCode ierr;
PetscFunctionBegin;
/* Create the factorization matrix F */
ierr = MatCreate(((PetscObject)A)->comm,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(B,1,0,PETSC_NULL,0,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscNewLog(B,Mat_Spooles,&lu);CHKERRQ(ierr);
B->spptr = lu;
lu->flg = DIFFERENT_NONZERO_PATTERN;
lu->options.useQR = PETSC_FALSE;
if (ftype == MAT_FACTOR_CHOLESKY) {
B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_MPISBAIJSpooles;
B->ops->view = MatView_Spooles;
B->ops->destroy = MatDestroy_MPISBAIJSpooles;
ierr = PetscObjectComposeFunctionDynamic((PetscObject)B,"MatFactorGetSolverPackage_C","MatFactorGetSolverPackage_mpisbaij_spooles",MatFactorGetSolverPackage_mpisbaij_spooles);CHKERRQ(ierr);
lu->options.symflag = SPOOLES_SYMMETRIC;
lu->options.pivotingflag = SPOOLES_NO_PIVOTING;
} else SETERRQ(PETSC_ERR_SUP,"Only Cholesky for SBAIJ matrices, use AIJ for LU");
B->factor = ftype;
ierr = MPI_Comm_dup(((PetscObject)A)->comm,&(lu->comm_spooles));CHKERRQ(ierr);
*F = B;
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Vec x,y,u,s1,s2;
Mat A,sA,sB;
PetscRandom rctx;
PetscReal r1,r2,rnorm,tol = PETSC_SQRT_MACHINE_EPSILON;
PetscScalar one=1.0, neg_one=-1.0, value[3], four=4.0,alpha=0.1;
PetscInt n,col[3],n1,block,row,i,j,i2,j2,Ii,J,rstart,rend,bs=1,mbs=16,d_nz=3,o_nz=3,prob=2;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscBool flg;
MatType type;
ierr = PetscInitialize(&argc,&args,(char*)0,help);
if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-mbs",&mbs,NULL);
CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);
CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);
CHKERRQ(ierr);
n = mbs*bs;
/* Assemble MPISBAIJ matrix sA */
ierr = MatCreate(PETSC_COMM_WORLD,&sA);
CHKERRQ(ierr);
ierr = MatSetSizes(sA,PETSC_DECIDE,PETSC_DECIDE,n,n);
CHKERRQ(ierr);
ierr = MatSetType(sA,MATSBAIJ);
CHKERRQ(ierr);
ierr = MatSetFromOptions(sA);
CHKERRQ(ierr);
ierr = MatGetType(sA,&type);
CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(sA,bs,d_nz,NULL,o_nz,NULL);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(sA,bs,d_nz,NULL);
CHKERRQ(ierr);
ierr = MatSetOption(sA,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);
CHKERRQ(ierr);
if (bs == 1) {
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0;
value[1] = 2.0;
value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = n - 1;
col[0]=0;
col[1] = n - 2;
col[2] = n - 1;
value[0]= 0.1;
value[1]=-1;
value[2]=2;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0;
col[0] = 0;
col[1] = 1;
col[2]=n-1;
value[0] = 2.0;
value[1] = -1.0;
value[2]=0.1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (int) PetscSqrtReal((PetscReal)n);
if (n1*n1 != n) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"n must be a perfect square of n1");
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
ierr = MatSetValues(sA,1,&Ii,1,&Ii,&four,INSERT_VALUES);
CHKERRQ(ierr);
}
}
}
/* end of if (bs == 1) */
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0;
value[1] = 4.0;
value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = bs - 1+block*bs;
col[0] = bs - 2+block*bs;
col[1] = bs - 1+block*bs;
value[0]=-1.0;
value[1]=4.0;
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0+block*bs;
col[0] = 0+block*bs;
col[1] = 1+block*bs;
value[0]=4.0;
value[1] = -1.0;
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(sA,1,&i,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
col[0]=i;
row=i+bs;
ierr = MatSetValues(sA,1,&row,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(sA,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(sA,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
/* Test MatView() */
ierr = MatCreateBAIJ(PETSC_COMM_WORLD,bs,PETSC_DECIDE,PETSC_DECIDE,n,n,d_nz,NULL,o_nz,NULL,&A);
CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);
CHKERRQ(ierr);
if (bs == 1) {
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0;
value[1] = 2.0;
value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = n - 1;
col[0]=0;
col[1] = n - 2;
col[2] = n - 1;
value[0]= 0.1;
value[1]=-1;
value[2]=2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0;
col[0] = 0;
col[1] = 1;
col[2]=n-1;
value[0] = 2.0;
value[1] = -1.0;
value[2]=0.1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (int) PetscSqrtReal((PetscReal)n);
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
ierr = MatSetValues(A,1,&Ii,1,&Ii,&four,INSERT_VALUES);
CHKERRQ(ierr);
}
}
}
/* end of if (bs == 1) */
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0;
value[1] = 4.0;
value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = bs - 1+block*bs;
col[0] = bs - 2+block*bs;
col[1] = bs - 1+block*bs;
value[0]=-1.0;
value[1]=4.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0+block*bs;
col[0] = 0+block*bs;
col[1] = 1+block*bs;
value[0]=4.0;
value[1] = -1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(A,1,&i,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
col[0]=i;
row=i+bs;
ierr = MatSetValues(A,1,&row,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
/* Test MatGetSize(), MatGetLocalSize() */
ierr = MatGetSize(sA, &i,&j);
CHKERRQ(ierr);
ierr = MatGetSize(A, &i2,&j2);
CHKERRQ(ierr);
i -= i2;
j -= j2;
if (i || j) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatGetSize()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatGetLocalSize(sA, &i,&j);
CHKERRQ(ierr);
ierr = MatGetLocalSize(A, &i2,&j2);
CHKERRQ(ierr);
i2 -= i;
j2 -= j;
if (i2 || j2) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatGetLocalSize()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* vectors */
/*--------------------*/
/* i is obtained from MatGetLocalSize() */
ierr = VecCreate(PETSC_COMM_WORLD,&x);
CHKERRQ(ierr);
ierr = VecSetSizes(x,i,PETSC_DECIDE);
CHKERRQ(ierr);
ierr = VecSetFromOptions(x);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&s1);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&s2);
CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);
CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);
CHKERRQ(ierr);
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = VecSet(u,one);
CHKERRQ(ierr);
/* Test MatNorm() */
ierr = MatNorm(A,NORM_FROBENIUS,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_FROBENIUS,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_FROBENIUS(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_INFINITY,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_INFINITY,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: MatNorm_INFINITY(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_1,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_1,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: MatNorm_1(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
/* Test MatGetOwnershipRange() */
ierr = MatGetOwnershipRange(sA,&rstart,&rend);
CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&i2,&j2);
CHKERRQ(ierr);
i2 -= rstart;
j2 -= rend;
if (i2 || j2) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MaGetOwnershipRange()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* Test MatDiagonalScale() */
ierr = MatDiagonalScale(A,x,x);
CHKERRQ(ierr);
ierr = MatDiagonalScale(sA,x,x);
CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDiagonalScale");
/* Test MatGetDiagonal(), MatScale() */
ierr = MatGetDiagonal(A,s1);
CHKERRQ(ierr);
ierr = MatGetDiagonal(sA,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDiagonalScale() or MatGetDiagonal(), r1=%g \n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatScale(A,alpha);
CHKERRQ(ierr);
ierr = MatScale(sA,alpha);
CHKERRQ(ierr);
/* Test MatGetRowMaxAbs() */
ierr = MatGetRowMaxAbs(A,s1,NULL);
CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(sA,s2,NULL);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetRowMaxAbs() \n");
CHKERRQ(ierr);
}
/* Test MatMult(), MatMultAdd() */
ierr = MatMultEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMult() or MatScale()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatMultAddEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMultAdd()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* Test MatMultTranspose(), MatMultTransposeAdd() */
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = MatMultTranspose(A,x,s1);
CHKERRQ(ierr);
ierr = MatMultTranspose(sA,x,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMult() or MatScale(), err=%g\n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
}
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = VecSetRandom(y,rctx);
CHKERRQ(ierr);
ierr = MatMultTransposeAdd(A,x,y,s1);
CHKERRQ(ierr);
ierr = MatMultTransposeAdd(sA,x,y,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMultAdd(), err=%g \n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
}
/* Test MatDuplicate() */
ierr = MatDuplicate(sA,MAT_COPY_VALUES,&sB);
CHKERRQ(ierr);
ierr = MatEqual(sA,sB,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD," Error in MatDuplicate(), sA != sB \n");
CHKERRQ(ierr);
CHKERRQ(ierr);
}
ierr = MatMultEqual(sA,sB,5,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDuplicate() or MatMult()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatMultAddEqual(sA,sB,5,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDuplicate() or MatMultAdd(()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatDestroy(&sB);
CHKERRQ(ierr);
ierr = VecDestroy(&u);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&y);
CHKERRQ(ierr);
ierr = VecDestroy(&s1);
CHKERRQ(ierr);
ierr = VecDestroy(&s2);
CHKERRQ(ierr);
ierr = MatDestroy(&sA);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);
CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PETSC_EXTERN PetscErrorCode MatISSetMPIXAIJPreallocation_Private(Mat A, Mat B, PetscBool maxreduce)
{
Mat_IS *matis = (Mat_IS*)(A->data);
PetscInt *my_dnz,*my_onz,*dnz,*onz,*mat_ranges,*row_ownership;
const PetscInt *global_indices_r,*global_indices_c;
PetscInt i,j,bs,rows,cols;
PetscInt lrows,lcols;
PetscInt local_rows,local_cols;
PetscMPIInt nsubdomains;
PetscBool isdense,issbaij;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&nsubdomains);CHKERRQ(ierr);
ierr = MatGetSize(A,&rows,&cols);CHKERRQ(ierr);
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&local_rows,&local_cols);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr);
if (A->rmap->mapping != A->cmap->mapping) {
ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr);
} else {
global_indices_c = global_indices_r;
}
if (issbaij) {
ierr = MatGetRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
/*
An SF reduce is needed to sum up properly on shared rows.
Note that generally preallocation is not exact, since it overestimates nonzeros
*/
if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
ierr = MatISComputeSF_Private(A);CHKERRQ(ierr);
}
ierr = MatGetLocalSize(A,&lrows,&lcols);CHKERRQ(ierr);
ierr = MatPreallocateInitialize(PetscObjectComm((PetscObject)A),lrows,lcols,dnz,onz);CHKERRQ(ierr);
/* All processes need to compute entire row ownership */
ierr = PetscMalloc1(rows,&row_ownership);CHKERRQ(ierr);
ierr = MatGetOwnershipRanges(A,(const PetscInt**)&mat_ranges);CHKERRQ(ierr);
for (i=0;i<nsubdomains;i++) {
for (j=mat_ranges[i];j<mat_ranges[i+1];j++) {
row_ownership[j] = i;
}
}
/*
my_dnz and my_onz contains exact contribution to preallocation from each local mat
then, they will be summed up properly. This way, preallocation is always sufficient
*/
ierr = PetscCalloc2(local_rows,&my_dnz,local_rows,&my_onz);CHKERRQ(ierr);
/* preallocation as a MATAIJ */
if (isdense) { /* special case for dense local matrices */
for (i=0;i<local_rows;i++) {
PetscInt index_row = global_indices_r[i];
for (j=i;j<local_rows;j++) {
PetscInt owner = row_ownership[index_row];
PetscInt index_col = global_indices_c[j];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1;
} else { /* offdiag block */
my_onz[i] += 1;
}
/* same as before, interchanging rows and cols */
if (i != j) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[j] += 1;
} else {
my_onz[j] += 1;
}
}
}
}
} else { /* TODO: this could be optimized using MatGetRowIJ */
for (i=0;i<local_rows;i++) {
const PetscInt *cols;
PetscInt ncols,index_row = global_indices_r[i];
ierr = MatGetRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
PetscInt owner = row_ownership[index_row];
PetscInt index_col = global_indices_c[cols[j]];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1;
} else { /* offdiag block */
my_onz[i] += 1;
}
/* same as before, interchanging rows and cols */
if (issbaij && index_col != index_row) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[cols[j]] += 1;
} else {
my_onz[cols[j]] += 1;
}
}
}
ierr = MatRestoreRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr);
if (global_indices_c != global_indices_r) {
ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr);
}
ierr = PetscFree(row_ownership);CHKERRQ(ierr);
/* Reduce my_dnz and my_onz */
if (maxreduce) {
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr);
} else {
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr);
}
ierr = PetscFree2(my_dnz,my_onz);CHKERRQ(ierr);
/* Resize preallocation if overestimated */
for (i=0;i<lrows;i++) {
dnz[i] = PetscMin(dnz[i],lcols);
onz[i] = PetscMin(onz[i],cols-lcols);
}
/* set preallocation */
ierr = MatMPIAIJSetPreallocation(B,0,dnz,0,onz);CHKERRQ(ierr);
for (i=0;i<lrows/bs;i++) {
dnz[i] = dnz[i*bs]/bs;
onz[i] = onz[i*bs]/bs;
}
ierr = MatMPIBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatPreallocateFinalize(dnz,onz);CHKERRQ(ierr);
if (issbaij) {
ierr = MatRestoreRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode MatISGetMPIXAIJ_IS(Mat mat, MatReuse reuse, Mat *M)
{
Mat_IS *matis = (Mat_IS*)(mat->data);
/* info on mat */
/* ISLocalToGlobalMapping rmapping,cmapping; */
PetscInt bs,rows,cols;
PetscInt lrows,lcols;
PetscInt local_rows,local_cols;
PetscBool isdense,issbaij,issbaij_red;
/* values insertion */
PetscScalar *array;
PetscInt *local_indices,*global_indices;
/* work */
PetscInt i,j,index_row;
PetscErrorCode ierr;
PetscFunctionBegin;
/* MISSING CHECKS
- rectangular case not covered (it is not allowed by MATIS)
*/
/* get info from mat */
/* ierr = MatGetLocalToGlobalMapping(mat,&rmapping,&cmapping);CHKERRQ(ierr); */
ierr = MatGetSize(mat,&rows,&cols);CHKERRQ(ierr);
ierr = MatGetBlockSize(mat,&bs);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&local_rows,&local_cols);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
/* work */
ierr = PetscMalloc1(local_rows,&local_indices);CHKERRQ(ierr);
for (i=0;i<local_rows;i++) local_indices[i]=i;
/* map indices of local mat to global values */
ierr = PetscMalloc(PetscMax(local_cols,local_rows)*sizeof(*global_indices),&global_indices);CHKERRQ(ierr);
/* ierr = ISLocalToGlobalMappingApply(rmapping,local_rows,local_indices,global_indices);CHKERRQ(ierr); */
ierr = ISLocalToGlobalMappingApply(matis->mapping,local_rows,local_indices,global_indices);CHKERRQ(ierr);
if (issbaij) {
ierr = MatGetRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
if (reuse == MAT_INITIAL_MATRIX) {
Mat new_mat;
MatType new_mat_type;
Vec vec_dnz,vec_onz;
PetscScalar *my_dnz,*my_onz;
PetscInt *dnz,*onz,*mat_ranges,*row_ownership;
PetscInt index_col,owner;
PetscMPIInt nsubdomains;
/* determining new matrix type */
ierr = MPI_Allreduce(&issbaij,&issbaij_red,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)mat));CHKERRQ(ierr);
if (issbaij_red) {
new_mat_type = MATSBAIJ;
} else {
if (bs>1) {
new_mat_type = MATBAIJ;
} else {
new_mat_type = MATAIJ;
}
}
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&nsubdomains);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)mat),&new_mat);CHKERRQ(ierr);
ierr = MatSetSizes(new_mat,PETSC_DECIDE,PETSC_DECIDE,rows,cols);CHKERRQ(ierr);
ierr = MatSetBlockSize(new_mat,bs);CHKERRQ(ierr);
ierr = MatSetType(new_mat,new_mat_type);CHKERRQ(ierr);
ierr = MatSetUp(new_mat);CHKERRQ(ierr);
ierr = MatGetLocalSize(new_mat,&lrows,&lcols);CHKERRQ(ierr);
/*
preallocation
*/
ierr = MatPreallocateInitialize(PetscObjectComm((PetscObject)new_mat),lrows,lcols,dnz,onz);CHKERRQ(ierr);
/*
Some vectors are needed to sum up properly on shared interface dofs.
Preallocation macros cannot do the job.
Note that preallocation is not exact, since it overestimates nonzeros
*/
ierr = MatCreateVecs(new_mat,NULL,&vec_dnz);CHKERRQ(ierr);
/* ierr = VecSetLocalToGlobalMapping(vec_dnz,rmapping);CHKERRQ(ierr); */
ierr = VecSetLocalToGlobalMapping(vec_dnz,matis->mapping);CHKERRQ(ierr);
ierr = VecDuplicate(vec_dnz,&vec_onz);CHKERRQ(ierr);
/* All processes need to compute entire row ownership */
ierr = PetscMalloc1(rows,&row_ownership);CHKERRQ(ierr);
ierr = MatGetOwnershipRanges(new_mat,(const PetscInt**)&mat_ranges);CHKERRQ(ierr);
for (i=0;i<nsubdomains;i++) {
for (j=mat_ranges[i];j<mat_ranges[i+1];j++) {
row_ownership[j]=i;
}
}
/*
my_dnz and my_onz contains exact contribution to preallocation from each local mat
then, they will be summed up properly. This way, preallocation is always sufficient
*/
ierr = PetscMalloc1(local_rows,&my_dnz);CHKERRQ(ierr);
ierr = PetscMalloc1(local_rows,&my_onz);CHKERRQ(ierr);
ierr = PetscMemzero(my_dnz,local_rows*sizeof(*my_dnz));CHKERRQ(ierr);
ierr = PetscMemzero(my_onz,local_rows*sizeof(*my_onz));CHKERRQ(ierr);
/* preallocation as a MATAIJ */
if (isdense) { /* special case for dense local matrices */
for (i=0;i<local_rows;i++) {
index_row = global_indices[i];
for (j=i;j<local_rows;j++) {
owner = row_ownership[index_row];
index_col = global_indices[j];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1.0;
} else { /* offdiag block */
my_onz[i] += 1.0;
}
/* same as before, interchanging rows and cols */
if (i != j) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[j] += 1.0;
} else {
my_onz[j] += 1.0;
}
}
}
}
} else {
for (i=0;i<local_rows;i++) {
PetscInt ncols;
const PetscInt *cols;
index_row = global_indices[i];
ierr = MatGetRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
owner = row_ownership[index_row];
index_col = global_indices[cols[j]];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1.0;
} else { /* offdiag block */
my_onz[i] += 1.0;
}
/* same as before, interchanging rows and cols */
if (issbaij) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[j] += 1.0;
} else {
my_onz[j] += 1.0;
}
}
}
ierr = MatRestoreRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
}
}
ierr = VecSet(vec_dnz,0.0);CHKERRQ(ierr);
ierr = VecSet(vec_onz,0.0);CHKERRQ(ierr);
if (local_rows) { /* multilevel guard */
ierr = VecSetValuesLocal(vec_dnz,local_rows,local_indices,my_dnz,ADD_VALUES);CHKERRQ(ierr);
ierr = VecSetValuesLocal(vec_onz,local_rows,local_indices,my_onz,ADD_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(vec_dnz);CHKERRQ(ierr);
ierr = VecAssemblyBegin(vec_onz);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec_dnz);CHKERRQ(ierr);
ierr = VecAssemblyEnd(vec_onz);CHKERRQ(ierr);
ierr = PetscFree(my_dnz);CHKERRQ(ierr);
ierr = PetscFree(my_onz);CHKERRQ(ierr);
ierr = PetscFree(row_ownership);CHKERRQ(ierr);
/* set computed preallocation in dnz and onz */
ierr = VecGetArray(vec_dnz,&array);CHKERRQ(ierr);
for (i=0; i<lrows; i++) dnz[i] = (PetscInt)PetscRealPart(array[i]);
ierr = VecRestoreArray(vec_dnz,&array);CHKERRQ(ierr);
ierr = VecGetArray(vec_onz,&array);CHKERRQ(ierr);
for (i=0;i<lrows;i++) onz[i] = (PetscInt)PetscRealPart(array[i]);
ierr = VecRestoreArray(vec_onz,&array);CHKERRQ(ierr);
ierr = VecDestroy(&vec_dnz);CHKERRQ(ierr);
ierr = VecDestroy(&vec_onz);CHKERRQ(ierr);
/* Resize preallocation if overestimated */
for (i=0;i<lrows;i++) {
dnz[i] = PetscMin(dnz[i],lcols);
onz[i] = PetscMin(onz[i],cols-lcols);
}
/* set preallocation */
ierr = MatMPIAIJSetPreallocation(new_mat,0,dnz,0,onz);CHKERRQ(ierr);
for (i=0;i<lrows/bs;i++) {
dnz[i] = dnz[i*bs]/bs;
onz[i] = onz[i*bs]/bs;
}
ierr = MatMPIBAIJSetPreallocation(new_mat,bs,0,dnz,0,onz);CHKERRQ(ierr);
for (i=0;i<lrows/bs;i++) {
dnz[i] = dnz[i]-i;
}
ierr = MatMPISBAIJSetPreallocation(new_mat,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatPreallocateFinalize(dnz,onz);CHKERRQ(ierr);
*M = new_mat;
} else {
PetscInt mbs,mrows,mcols;
/* some checks */
ierr = MatGetBlockSize(*M,&mbs);CHKERRQ(ierr);
ierr = MatGetSize(*M,&mrows,&mcols);CHKERRQ(ierr);
if (mrows != rows) {
SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of rows (%d != %d)",rows,mrows);
}
if (mrows != rows) {
SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong number of cols (%d != %d)",cols,mcols);
}
if (mbs != bs) {
SETERRQ2(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot reuse matrix. Wrong block size (%d != %d)",bs,mbs);
}
ierr = MatZeroEntries(*M);CHKERRQ(ierr);
}
/* set local to global mappings */
/* ierr = MatSetLocalToGlobalMapping(*M,rmapping,cmapping);CHKERRQ(ierr); */
/* Set values */
if (isdense) { /* special case for dense local matrices */
ierr = MatSetOption(*M,MAT_ROW_ORIENTED,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatDenseGetArray(matis->A,&array);CHKERRQ(ierr);
ierr = MatSetValues(*M,local_rows,global_indices,local_cols,global_indices,array,ADD_VALUES);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(matis->A,&array);CHKERRQ(ierr);
ierr = PetscFree(local_indices);CHKERRQ(ierr);
ierr = PetscFree(global_indices);CHKERRQ(ierr);
} else { /* very basic values insertion for all other matrix types */
ierr = PetscFree(local_indices);CHKERRQ(ierr);
for (i=0;i<local_rows;i++) {
ierr = MatGetRow(matis->A,i,&j,(const PetscInt**)&local_indices,(const PetscScalar**)&array);CHKERRQ(ierr);
/* ierr = MatSetValuesLocal(*M,1,&i,j,local_indices,array,ADD_VALUES);CHKERRQ(ierr); */
ierr = ISLocalToGlobalMappingApply(matis->mapping,j,local_indices,global_indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(matis->mapping,1,&i,&index_row);CHKERRQ(ierr);
ierr = MatSetValues(*M,1,&index_row,j,global_indices,array,ADD_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(matis->A,i,&j,(const PetscInt**)&local_indices,(const PetscScalar**)&array);CHKERRQ(ierr);
}
ierr = PetscFree(global_indices);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(*M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (isdense) {
ierr = MatSetOption(*M,MAT_ROW_ORIENTED,PETSC_TRUE);CHKERRQ(ierr);
}
if (issbaij) {
ierr = MatRestoreRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
