PETSC_EXTERN PetscErrorCode MatGetFactor_seqsbaij_pastix(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
PetscErrorCode ierr;
Mat_Pastix *pastix;
PetscFunctionBegin;
if (ftype != MAT_FACTOR_CHOLESKY) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot use PETSc SBAIJ matrices with PaStiX LU, use AIJ matrix");
/* Create the factorization matrix */
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(B,1,0,NULL);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(B,1,0,NULL,0,NULL);CHKERRQ(ierr);
B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_SBAIJPASTIX;
B->ops->view = MatView_PaStiX;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_pastix);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_CHOLESKY;
ierr = PetscNewLog(B,&pastix);CHKERRQ(ierr);
pastix->CleanUpPastix = PETSC_FALSE;
pastix->isAIJ = PETSC_TRUE;
pastix->scat_rhs = NULL;
pastix->scat_sol = NULL;
pastix->Destroy = B->ops->destroy;
B->ops->destroy = MatDestroy_Pastix;
B->spptr = (void*)pastix;
*F = B;
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode MatGetFactor_seqsbaij_cholmod(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
Mat_CHOLMOD *chol;
PetscErrorCode ierr;
PetscInt m=A->rmap->n,n=A->cmap->n,bs;
PetscFunctionBegin;
if (ftype != MAT_FACTOR_CHOLESKY) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"CHOLMOD cannot do %s factorization with SBAIJ, only %s",
MatFactorTypes[ftype],MatFactorTypes[MAT_FACTOR_CHOLESKY]);
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
if (bs != 1) SETERRQ1(PetscObjectComm((PetscObject)A),PETSC_ERR_SUP,"CHOLMOD only supports block size=1, given %D",bs);
/* Create the factorization matrix F */
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,PETSC_DECIDE,PETSC_DECIDE,m,n);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(B,1,0,NULL);CHKERRQ(ierr);
ierr = PetscNewLog(B,&chol);CHKERRQ(ierr);
chol->Wrap = MatWrapCholmod_seqsbaij;
chol->Destroy = MatDestroy_SeqSBAIJ;
B->spptr = chol;
B->ops->view = MatView_CHOLMOD;
B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_CHOLMOD;
B->ops->destroy = MatDestroy_CHOLMOD;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_seqsbaij_cholmod);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_CHOLESKY;
B->assembled = PETSC_TRUE; /* required by -ksp_view */
B->preallocated = PETSC_TRUE;
ierr = CholmodStart(B);CHKERRQ(ierr);
*F = B;
PetscFunctionReturn(0);
}
int main(int argc, char **args)
{
Mat A;
MatPartitioning part;
IS is;
PetscInt i,m,N,rstart,rend,nemptyranks,*emptyranks,nbigranks,*bigranks;
PetscMPIInt rank,size;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
nemptyranks = 10;
nbigranks = 10;
ierr = PetscMalloc2(nemptyranks,PetscInt,&emptyranks,nbigranks,PetscInt,&bigranks);CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,PETSC_NULL,"Partitioning example options",PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsIntArray("-emptyranks","Ranks to be skipped by partition","",emptyranks,&nemptyranks,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsIntArray("-bigranks","Ranks to be overloaded","",bigranks,&nbigranks,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
m = 1;
for (i=0; i<nemptyranks; i++) if (rank == emptyranks[i]) m = 0;
for (i=0; i<nbigranks; i++) if (rank == bigranks[i]) m = 5;
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m,m,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,1,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,1,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,1,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,1,2,PETSC_NULL,1,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetSize(A,PETSC_NULL,&N);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
const PetscInt cols[] = {(i+N-1)%N,i,(i+1)%N};
const PetscScalar vals[] = {1,1,1};
ierr = MatSetValues(A,1,&i,3,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFree2(emptyranks,bigranks);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@
MatXAIJSetPreallocation - set preallocation for serial and parallel AIJ, BAIJ, and SBAIJ matrices
Collective on Mat
Input Arguments:
+ A - matrix being preallocated
. bs - block size
. dnnz - number of nonzero blocks per block row of diagonal part of parallel matrix
. onnz - number of nonzero blocks per block row of off-diagonal part of parallel matrix
. dnnzu - number of nonzero blocks per block row of upper-triangular part of diagonal part of parallel matrix
- onnzu - number of nonzero blocks per block row of upper-triangular part of off-diagonal part of parallel matrix
Level: beginner
.seealso: MatSeqAIJSetPreallocation(), MatMPIAIJSetPreallocation(), MatSeqBAIJSetPreallocation(), MatMPIBAIJSetPreallocation(), MatSeqSBAIJSetPreallocation(), MatMPISBAIJSetPreallocation(),
PetscSplitOwnership()
@*/
PetscErrorCode MatXAIJSetPreallocation(Mat A,PetscInt bs,const PetscInt dnnz[],const PetscInt onnz[],const PetscInt dnnzu[],const PetscInt onnzu[])
{
PetscErrorCode ierr;
void (*aij)(void);
PetscFunctionBegin;
ierr = MatSetBlockSize(A,bs);
CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->rmap);
CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->cmap);
CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,0,dnnz);
CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,0,dnnz,0,onnz);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,bs,0,dnnzu);
CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,bs,0,dnnzu,0,onnzu);
CHKERRQ(ierr);
/*
In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
good before going on with it.
*/
ierr = PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
CHKERRQ(ierr);
if (!aij) {
ierr = PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
CHKERRQ(ierr);
}
if (aij) {
if (bs == 1) {
ierr = MatSeqAIJSetPreallocation(A,0,dnnz);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz,0,onnz);
CHKERRQ(ierr);
} else { /* Convert block-row precallocation to scalar-row */
PetscInt i,m,*sdnnz,*sonnz;
ierr = MatGetLocalSize(A,&m,NULL);
CHKERRQ(ierr);
ierr = PetscMalloc2((!!dnnz)*m,PetscInt,&sdnnz,(!!onnz)*m,PetscInt,&sonnz);
CHKERRQ(ierr);
for (i=0; i<m; i++) {
if (dnnz) sdnnz[i] = dnnz[i/bs] * bs;
if (onnz) sonnz[i] = onnz[i/bs] * bs;
}
ierr = MatSeqAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL,0,onnz ? sonnz : NULL);
CHKERRQ(ierr);
ierr = PetscFree2(sdnnz,sonnz);
CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "MatConvert_MPIAIJ_MPISBAIJ"
PetscErrorCode MatConvert_MPIAIJ_MPISBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat)
{
PetscErrorCode ierr;
Mat M;
Mat_MPIAIJ *mpimat = (Mat_MPIAIJ*)A->data;
Mat_SeqAIJ *Aa = (Mat_SeqAIJ*)mpimat->A->data,*Ba = (Mat_SeqAIJ*)mpimat->B->data;
PetscInt *d_nnz,*o_nnz;
PetscInt i,j,nz;
PetscInt m,n,lm,ln;
PetscInt rstart,rend;
const PetscScalar *vwork;
const PetscInt *cwork;
PetscFunctionBegin;
if (!A->symmetric) SETERRQ(((PetscObject)A)->comm,PETSC_ERR_USER,"Matrix must be symmetric. Call MatSetOption(mat,MAT_SYMMETRIC,PETSC_TRUE)");
ierr = MatGetSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&lm,&ln);CHKERRQ(ierr);
ierr = PetscMalloc2(lm,PetscInt,&d_nnz,lm,PetscInt,&o_nnz);CHKERRQ(ierr);
ierr = MatMarkDiagonal_SeqAIJ(mpimat->A);CHKERRQ(ierr);
for (i=0;i<lm;i++){
d_nnz[i] = Aa->i[i+1] - Aa->diag[i];
o_nnz[i] = Ba->i[i+1] - Ba->i[i];
}
ierr = MatCreate(((PetscObject)A)->comm,&M);CHKERRQ(ierr);
ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr);
ierr = MatSetType(M,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(M,1,0,d_nnz);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(M,1,0,d_nnz,0,o_nnz);CHKERRQ(ierr);
ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart;i<rend;i++){
ierr = MatGetRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
j = 0;
while (cwork[j] < i){ j++; nz--;}
ierr = MatSetValues(M,1,&i,nz,cwork+j,vwork+j,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_REUSE_MATRIX) {
ierr = MatHeaderReplace(A,M);CHKERRQ(ierr);
} else {
*newmat = M;
}
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode MatConvert_MPIBAIJ_MPISBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat)
{
PetscErrorCode ierr;
Mat M;
Mat_MPIBAIJ *mpimat = (Mat_MPIBAIJ*)A->data;
Mat_SeqBAIJ *Aa = (Mat_SeqBAIJ*)mpimat->A->data,*Ba = (Mat_SeqBAIJ*)mpimat->B->data;
PetscInt *d_nnz,*o_nnz;
PetscInt i,j,nz;
PetscInt m,n,lm,ln;
PetscInt rstart,rend;
const PetscScalar *vwork;
const PetscInt *cwork;
PetscInt bs = A->rmap->bs;
PetscFunctionBegin;
ierr = MatGetSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&lm,&ln);CHKERRQ(ierr);
ierr = PetscMalloc2(lm/bs,&d_nnz,lm/bs,&o_nnz);CHKERRQ(ierr);
ierr = MatMarkDiagonal_SeqBAIJ(mpimat->A);CHKERRQ(ierr);
for (i=0; i<lm/bs; i++) {
d_nnz[i] = Aa->i[i+1] - Aa->diag[i];
o_nnz[i] = Ba->i[i+1] - Ba->i[i];
}
ierr = MatCreate(PetscObjectComm((PetscObject)A),&M);CHKERRQ(ierr);
ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr);
ierr = MatSetType(M,MATMPISBAIJ);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(M,bs,0,d_nnz);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(M,bs,0,d_nnz,0,o_nnz);CHKERRQ(ierr);
ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatSetOption(M,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatGetRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
j = 0;
ierr = MatSetValues(M,1,&i,nz,cwork+j,vwork+j,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(A,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_REUSE_MATRIX) {
ierr = MatHeaderReplace(A,&M);CHKERRQ(ierr);
} else {
*newmat = M;
}
PetscFunctionReturn(0);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscSF sf;
PetscInt bs,i,nroots,*rootdata,nleaves,*leafdata,nlocalcols;
const PetscInt *gidxs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) {
SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
}
ierr = MatGetLocalSize(B,&nroots,NULL);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&nleaves,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscCalloc2(nroots,&rootdata,nleaves,&leafdata);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)B),&sf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,B->rmap,nleaves,NULL,PETSC_COPY_VALUES,gidxs);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
if (!d_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += d_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += o_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves;i++) {
leafdata[i] = PetscMin(leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree2(rootdata,leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatCreateMPISBSTRM(MPI_Comm comm,PetscInt bs,PetscInt m,PetscInt n,PetscInt M,PetscInt N,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[],Mat *A)
{
PetscErrorCode ierr;
PetscMPIInt size;
PetscFunctionBegin;
ierr = MatCreate(comm,A);CHKERRQ(ierr);
ierr = MatSetSizes(*A,m,n,M,N);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size > 1) {
ierr = MatSetType(*A,MATMPISBSTRM);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(*A,bs,d_nz,d_nnz,o_nz,o_nnz);CHKERRQ(ierr);
} else {
ierr = MatSetType(*A,MATSEQSBSTRM);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(*A,bs,d_nz,d_nnz);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscInt bs,i,nlocalcols;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
ierr = MatISComputeSF_Private(B);CHKERRQ(ierr);
}
if (!d_nnz) {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nz;
} else {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nz;
} else {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,NULL,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves;i++) {
matis->sf_leafdata[i] = PetscMin(matis->sf_leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves/bs;i++) {
matis->sf_leafdata[i] = matis->sf_leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves/bs;i++) {
matis->sf_leafdata[i] = matis->sf_leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_INTERN PetscErrorCode MatGetFactor_seqaij_bas(Mat A,MatFactorType ftype,Mat *B)
{
PetscInt n = A->rmap->n;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MatCreate(PetscObjectComm((PetscObject)A),B);CHKERRQ(ierr);
ierr = MatSetSizes(*B,n,n,n,n);CHKERRQ(ierr);
if (ftype == MAT_FACTOR_ICC) {
ierr = MatSetType(*B,MATSEQSBAIJ);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(*B,1,MAT_SKIP_ALLOCATION,NULL);CHKERRQ(ierr);
(*B)->ops->iccfactorsymbolic = MatICCFactorSymbolic_SeqAIJ_Bas;
(*B)->ops->choleskyfactornumeric = MatCholeskyFactorNumeric_SeqAIJ_Bas;
ierr = PetscObjectComposeFunction((PetscObject)*B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_seqaij_bas);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Factor type not supported");
(*B)->factortype = ftype;
ierr = PetscFree((*B)->solvertype);CHKERRQ(ierr);
ierr = PetscStrallocpy(MATSOLVERBAS,&(*B)->solvertype);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
PetscMPIInt size;
PetscErrorCode ierr;
Vec x,y,b,s1,s2;
Mat A; /* linear system matrix */
Mat sA,sB,sC; /* symmetric part of the matrices */
PetscInt n,mbs=16,bs=1,nz=3,prob=1,i,j,k1,k2,col[3],lf,block, row,Ii,J,n1,inc;
PetscReal norm1,norm2,rnorm,tol=PETSC_SMALL;
PetscScalar neg_one = -1.0,four=4.0,value[3];
IS perm, iscol;
PetscRandom rdm;
PetscBool doIcc=PETSC_TRUE,equal;
MatInfo minfo1,minfo2;
MatFactorInfo factinfo;
MatType type;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"This is a uniprocessor example only!");
ierr = PetscOptionsGetInt(NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-mbs",&mbs,NULL);CHKERRQ(ierr);
n = mbs*bs;
ierr = MatCreate(PETSC_COMM_SELF,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(A,MATSEQBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,nz,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_SELF,&sA);CHKERRQ(ierr);
ierr = MatSetSizes(sA,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(sA,MATSEQSBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(sA);CHKERRQ(ierr);
ierr = MatGetType(sA,&type);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)sA,MATSEQSBAIJ,&doIcc);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(sA,bs,nz,NULL);CHKERRQ(ierr);
ierr = MatSetOption(sA,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
/* Test MatGetOwnershipRange() */
ierr = MatGetOwnershipRange(A,&Ii,&J);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(sA,&i,&j);CHKERRQ(ierr);
if (i-Ii || j-J) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetOwnershipRange() in MatSBAIJ format\n");CHKERRQ(ierr);
}
/* Assemble matrix */
if (bs == 1) {
ierr = PetscOptionsGetInt(NULL,"-test_problem",&prob,NULL);CHKERRQ(ierr);
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0]=0; col[1] = n - 2; col[2] = n - 1;
value[0]= 0.1; value[1]=-1; value[2]=2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0;
col[0] = n-1; col[1] = 1; col[2] = 0;
value[0] = 0.1; value[1] = -1.0; value[2] = 2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (PetscInt) (PetscSqrtReal((PetscReal)n) + 0.001);
if (n1*n1 - n) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"sqrt(n) must be a positive interger!");
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatSetValues(A,1,&Ii,1,&Ii,&four,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&Ii,&four,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0; value[1] = 4.0; value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = bs - 1+block*bs; col[0] = bs - 2+block*bs; col[1] = bs - 1+block*bs;
value[0]=-1.0; value[1]=4.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0+block*bs; col[0] = 0+block*bs; col[1] = 1+block*bs;
value[0]=4.0; value[1] = -1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(A,1,&i,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
col[0]=i; row=i+bs;
ierr = MatSetValues(A,1,&row,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&row,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyBegin(sA,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(sA,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatGetInfo() of A and sA */
ierr = MatGetInfo(A,MAT_LOCAL,&minfo1);CHKERRQ(ierr);
ierr = MatGetInfo(sA,MAT_LOCAL,&minfo2);CHKERRQ(ierr);
/*
printf("A matrix nonzeros (BAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo1.nz_used,(int)minfo1.nz_allocated);
printf("sA matrix nonzeros(SBAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo2.nz_used,(int)minfo2.nz_allocated);
*/
i = (int) (minfo1.nz_used - minfo2.nz_used);
j = (int) (minfo1.nz_allocated - minfo2.nz_allocated);
k1 = (int) (minfo1.nz_allocated - minfo1.nz_used);
k2 = (int) (minfo2.nz_allocated - minfo2.nz_used);
if (i < 0 || j < 0 || k1 < 0 || k2 < 0) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error (compare A and sA): MatGetInfo()\n");CHKERRQ(ierr);
}
/* Test MatDuplicate() */
ierr = MatNorm(A,NORM_FROBENIUS,&norm1);CHKERRQ(ierr);
ierr = MatDuplicate(sA,MAT_COPY_VALUES,&sB);CHKERRQ(ierr);
ierr = MatEqual(sA,sB,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDuplicate()");
/* Test MatNorm() */
ierr = MatNorm(A,NORM_FROBENIUS,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_FROBENIUS,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_FROBENIUS, NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_INFINITY,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_INFINITY,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_INFINITY(), NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_1,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_1,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_INFINITY(), NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
/* Test MatGetInfo(), MatGetSize(), MatGetBlockSize() */
ierr = MatGetInfo(A,MAT_LOCAL,&minfo1);CHKERRQ(ierr);
ierr = MatGetInfo(sB,MAT_LOCAL,&minfo2);CHKERRQ(ierr);
/*
printf("matrix nonzeros (BAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo1.nz_used,(int)minfo1.nz_allocated);
printf("matrix nonzeros(SBAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo2.nz_used,(int)minfo2.nz_allocated);
*/
i = (int) (minfo1.nz_used - minfo2.nz_used);
j = (int) (minfo1.nz_allocated - minfo2.nz_allocated);
k1 = (int) (minfo1.nz_allocated - minfo1.nz_used);
k2 = (int) (minfo2.nz_allocated - minfo2.nz_used);
if (i < 0 || j < 0 || k1 < 0 || k2 < 0) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error(compare A and sB): MatGetInfo()\n");CHKERRQ(ierr);
}
ierr = MatGetSize(A,&Ii,&J);CHKERRQ(ierr);
ierr = MatGetSize(sB,&i,&j);CHKERRQ(ierr);
if (i-Ii || j-J) {
PetscPrintf(PETSC_COMM_SELF,"Error: MatGetSize()\n");CHKERRQ(ierr);
}
ierr = MatGetBlockSize(A, &Ii);CHKERRQ(ierr);
ierr = MatGetBlockSize(sB, &i);CHKERRQ(ierr);
if (i-Ii) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetBlockSize()\n");CHKERRQ(ierr);
}
ierr = PetscRandomCreate(PETSC_COMM_SELF,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(x,&s2);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
/* Test MatDiagonalScale(), MatGetDiagonal(), MatScale() */
#if !defined(PETSC_USE_COMPLEX)
/* Scaling matrix with complex numbers results non-spd matrix,
causing crash of MatForwardSolve() and MatBackwardSolve() */
ierr = MatDiagonalScale(A,x,x);CHKERRQ(ierr);
ierr = MatDiagonalScale(sB,x,x);CHKERRQ(ierr);
ierr = MatMultEqual(A,sB,10,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDiagonalScale");
ierr = MatGetDiagonal(A,s1);CHKERRQ(ierr);
ierr = MatGetDiagonal(sB,s2);CHKERRQ(ierr);
ierr = VecAXPY(s2,neg_one,s1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm1);CHKERRQ(ierr);
if (norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatGetDiagonal(), ||s1-s2||=%G\n",norm1);CHKERRQ(ierr);
}
{
PetscScalar alpha=0.1;
ierr = MatScale(A,alpha);CHKERRQ(ierr);
ierr = MatScale(sB,alpha);CHKERRQ(ierr);
}
#endif
/* Test MatGetRowMaxAbs() */
ierr = MatGetRowMaxAbs(A,s1,NULL);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(sB,s2,NULL);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatGetRowMaxAbs() \n");CHKERRQ(ierr);
}
/* Test MatMult() */
for (i=0; i<40; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(A,x,s1);CHKERRQ(ierr);
ierr = MatMult(sB,x,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMult(), norm1-norm2: %G\n",norm1);CHKERRQ(ierr);
}
}
/* MatMultAdd() */
for (i=0; i<40; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = VecSetRandom(y,rdm);CHKERRQ(ierr);
ierr = MatMultAdd(A,x,y,s1);CHKERRQ(ierr);
ierr = MatMultAdd(sB,x,y,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMultAdd(), norm1-norm2: %G\n",norm1);CHKERRQ(ierr);
}
}
/* Test MatCholeskyFactor(), MatICCFactor() with natural ordering */
ierr = MatGetOrdering(A,MATORDERINGNATURAL,&perm,&iscol);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
norm1 = tol;
inc = bs;
/* initialize factinfo */
ierr = PetscMemzero(&factinfo,sizeof(MatFactorInfo));CHKERRQ(ierr);
for (lf=-1; lf<10; lf += inc) {
if (lf==-1) { /* Cholesky factor of sB (duplicate sA) */
factinfo.fill = 5.0;
ierr = MatGetFactor(sB,MATSOLVERPETSC,MAT_FACTOR_CHOLESKY,&sC);CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(sC,sB,perm,&factinfo);CHKERRQ(ierr);
} else if (!doIcc) break;
else { /* incomplete Cholesky factor */
factinfo.fill = 5.0;
factinfo.levels = lf;
ierr = MatGetFactor(sB,MATSOLVERPETSC,MAT_FACTOR_ICC,&sC);CHKERRQ(ierr);
ierr = MatICCFactorSymbolic(sC,sB,perm,&factinfo);CHKERRQ(ierr);
}
ierr = MatCholeskyFactorNumeric(sC,sB,&factinfo);CHKERRQ(ierr);
/* MatView(sC, PETSC_VIEWER_DRAW_WORLD); */
/* test MatGetDiagonal on numeric factor */
/*
if (lf == -1) {
ierr = MatGetDiagonal(sC,s1);CHKERRQ(ierr);
printf(" in ex74.c, diag: \n");
ierr = VecView(s1,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
*/
ierr = MatMult(sB,x,b);CHKERRQ(ierr);
/* test MatForwardSolve() and MatBackwardSolve() */
if (lf == -1) {
ierr = MatForwardSolve(sC,b,s1);CHKERRQ(ierr);
ierr = MatBackwardSolve(sC,s1,s2);CHKERRQ(ierr);
ierr = VecAXPY(s2,neg_one,x);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&norm2);CHKERRQ(ierr);
if (10*norm1 < norm2) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatForwardSolve and BackwardSolve: Norm of error=%G, bs=%d\n",norm2,bs);CHKERRQ(ierr);
}
}
/* test MatSolve() */
ierr = MatSolve(sC,b,y);CHKERRQ(ierr);
ierr = MatDestroy(&sC);CHKERRQ(ierr);
/* Check the error */
ierr = VecAXPY(y,neg_one,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm2);CHKERRQ(ierr);
/* printf("lf: %d, error: %G\n", lf,norm2); */
if (10*norm1 < norm2 && lf-inc != -1) {
ierr = PetscPrintf(PETSC_COMM_SELF,"lf=%D, %D, Norm of error=%G, %G\n",lf-inc,lf,norm1,norm2);CHKERRQ(ierr);
}
norm1 = norm2;
if (norm2 < tol && lf != -1) break;
}
ierr = ISDestroy(&perm);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sB);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PETSC_INTERN PetscErrorCode MatConvert_SeqBAIJ_SeqSBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat B;
Mat_SeqBAIJ *a = (Mat_SeqBAIJ*)A->data;
Mat_SeqSBAIJ *b;
PetscErrorCode ierr;
PetscInt *ai=a->i,*aj,m=A->rmap->N,n=A->cmap->n,i,j,k,*bi,*bj,*browlengths;
PetscInt bs =A->rmap->bs,bs2=bs*bs,mbs=m/bs,dd;
MatScalar *av,*bv;
PetscBool flg;
PetscFunctionBegin;
if (!A->symmetric) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_USER,"Matrix must be symmetric. Call MatSetOption(mat,MAT_SYMMETRIC,PETSC_TRUE)");
if (n != m) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Matrix must be square");
ierr = MatMissingDiagonal_SeqBAIJ(A,&flg,&dd);
CHKERRQ(ierr); /* check for missing diagonals, then mark diag */
if (flg) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Matrix is missing diagonal %D",dd);
ierr = PetscMalloc1(mbs,&browlengths);
CHKERRQ(ierr);
for (i=0; i<mbs; i++) {
browlengths[i] = ai[i+1] - a->diag[i];
}
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);
CHKERRQ(ierr);
ierr = MatSetSizes(B,m,n,m,n);
CHKERRQ(ierr);
ierr = MatSetType(B,MATSEQSBAIJ);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(B,bs,0,browlengths);
CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_ROW_ORIENTED,PETSC_TRUE);
CHKERRQ(ierr);
b = (Mat_SeqSBAIJ*)(B->data);
bi = b->i;
bj = b->j;
bv = b->a;
bi[0] = 0;
for (i=0; i<mbs; i++) {
aj = a->j + a->diag[i];
av = a->a + (a->diag[i])*bs2;
for (j=0; j<browlengths[i]; j++) {
*bj = *aj;
bj++;
aj++;
for (k=0; k<bs2; k++) {
*bv = *av;
bv++;
av++;
}
}
bi[i+1] = bi[i] + browlengths[i];
b->ilen[i] = browlengths[i];
}
ierr = PetscFree(browlengths);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
if (reuse == MAT_INPLACE_MATRIX) {
ierr = MatHeaderReplace(A,&B);
CHKERRQ(ierr);
} else {
*newmat = B;
}
PetscFunctionReturn(0);
}
PETSC_INTERN PetscErrorCode MatConvert_SeqAIJ_SeqSBAIJ(Mat A,MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat B;
Mat_SeqAIJ *a = (Mat_SeqAIJ*)A->data;
Mat_SeqSBAIJ *b;
PetscErrorCode ierr;
PetscInt *ai=a->i,*aj,m=A->rmap->N,n=A->cmap->N,i,j,*bi,*bj,*rowlengths;
MatScalar *av,*bv;
PetscFunctionBegin;
if (!A->symmetric) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_USER,"Matrix must be symmetric. Call MatSetOption(mat,MAT_SYMMETRIC,PETSC_TRUE)");
if (n != m) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Matrix must be square");
ierr = PetscMalloc1(m,&rowlengths);
CHKERRQ(ierr);
for (i=0; i<m; i++) {
rowlengths[i] = ai[i+1] - a->diag[i];
}
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);
CHKERRQ(ierr);
ierr = MatSetSizes(B,m,n,m,n);
CHKERRQ(ierr);
ierr = MatSetType(B,MATSEQSBAIJ);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(B,1,0,rowlengths);
CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_ROW_ORIENTED,PETSC_TRUE);
CHKERRQ(ierr);
b = (Mat_SeqSBAIJ*)(B->data);
bi = b->i;
bj = b->j;
bv = b->a;
bi[0] = 0;
for (i=0; i<m; i++) {
aj = a->j + a->diag[i];
av = a->a + a->diag[i];
for (j=0; j<rowlengths[i]; j++) {
*bj = *aj;
bj++;
aj++;
*bv = *av;
bv++;
av++;
}
bi[i+1] = bi[i] + rowlengths[i];
b->ilen[i] = rowlengths[i];
}
ierr = PetscFree(rowlengths);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
if (reuse == MAT_INPLACE_MATRIX) {
ierr = MatHeaderReplace(A,&B);
CHKERRQ(ierr);
} else {
*newmat = B;
}
PetscFunctionReturn(0);
}
static PetscErrorCode ComputeSubdomainMatrix(DomainData dd, GLLData glldata, Mat *local_mat)
{
PetscErrorCode ierr;
PetscInt localsize,zloc,yloc,xloc,auxnex,auxney,auxnez;
PetscInt ie,je,ke,i,j,k,ig,jg,kg,ii,ming;
PetscInt *indexg,*cols,*colsg;
PetscScalar *vals;
Mat temp_local_mat,elem_mat_DBC=0,*usedmat;
IS submatIS;
PetscFunctionBeginUser;
ierr = MatGetSize(glldata.elem_mat,&i,&j);CHKERRQ(ierr);
ierr = PetscMalloc1(i,&indexg);CHKERRQ(ierr);
ierr = PetscMalloc1(i,&colsg);CHKERRQ(ierr);
/* get submatrix of elem_mat without dirichlet nodes */
if (!dd.pure_neumann && !dd.DBC_zerorows && !dd.ipx) {
xloc = dd.p+1;
yloc = 1;
zloc = 1;
if (dd.dim>1) yloc = dd.p+1;
if (dd.dim>2) zloc = dd.p+1;
ii = 0;
for (k=0;k<zloc;k++) {
for (j=0;j<yloc;j++) {
for (i=1;i<xloc;i++) {
indexg[ii]=k*xloc*yloc+j*xloc+i;
ii++;
}
}
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,ii,indexg,PETSC_COPY_VALUES,&submatIS);CHKERRQ(ierr);
ierr = MatGetSubMatrix(glldata.elem_mat,submatIS,submatIS,MAT_INITIAL_MATRIX,&elem_mat_DBC);CHKERRQ(ierr);
ierr = ISDestroy(&submatIS);CHKERRQ(ierr);
}
/* Assemble subdomain matrix */
localsize = dd.xm_l*dd.ym_l*dd.zm_l;
ierr = MatCreate(PETSC_COMM_SELF,&temp_local_mat);CHKERRQ(ierr);
ierr = MatSetSizes(temp_local_mat,localsize,localsize,localsize,localsize);CHKERRQ(ierr);
ierr = MatSetOptionsPrefix(temp_local_mat,"subdomain_");CHKERRQ(ierr);
/* set local matrices type: here we use SEQSBAIJ primarily for testing purpose */
/* in order to avoid conversions inside the BDDC code, use SeqAIJ if possible */
if (dd.DBC_zerorows && !dd.ipx) { /* in this case, we need to zero out some of the rows, so use seqaij */
ierr = MatSetType(temp_local_mat,MATSEQAIJ);CHKERRQ(ierr);
} else {
ierr = MatSetType(temp_local_mat,MATSEQSBAIJ);CHKERRQ(ierr);
}
ierr = MatSetFromOptions(temp_local_mat);CHKERRQ(ierr);
i = PetscPowRealInt(3.0*(dd.p+1.0),dd.dim);
ierr = MatSeqAIJSetPreallocation(temp_local_mat,i,NULL);CHKERRQ(ierr); /* very overestimated */
ierr = MatSeqSBAIJSetPreallocation(temp_local_mat,1,i,NULL);CHKERRQ(ierr); /* very overestimated */
ierr = MatSetOption(temp_local_mat,MAT_KEEP_NONZERO_PATTERN,PETSC_TRUE);CHKERRQ(ierr);
yloc = dd.p+1;
zloc = dd.p+1;
if (dd.dim < 3) zloc = 1;
if (dd.dim < 2) yloc = 1;
auxnez = dd.nez_l;
auxney = dd.ney_l;
auxnex = dd.nex_l;
if (dd.dim < 3) auxnez = 1;
if (dd.dim < 2) auxney = 1;
for (ke=0; ke<auxnez; ke++) {
for (je=0; je<auxney; je++) {
for (ie=0; ie<auxnex; ie++) {
/* customize element accounting for BC */
xloc = dd.p+1;
ming = 0;
usedmat = &glldata.elem_mat;
if (!dd.pure_neumann && !dd.DBC_zerorows && !dd.ipx) {
if (ie == 0) {
xloc = dd.p;
usedmat = &elem_mat_DBC;
} else {
ming = -1;
usedmat = &glldata.elem_mat;
}
}
/* local to the element/global to the subdomain indexing */
for (k=0; k<zloc; k++) {
kg = ke*dd.p+k;
for (j=0; j<yloc; j++) {
jg = je*dd.p+j;
for (i=0; i<xloc; i++) {
ig = ie*dd.p+i+ming;
ii = k*xloc*yloc+j*xloc+i;
indexg[ii] = kg*dd.xm_l*dd.ym_l+jg*dd.xm_l+ig;
}
}
}
/* Set values */
for (i=0; i<xloc*yloc*zloc; i++) {
ierr = MatGetRow(*usedmat,i,&j,(const PetscInt**)&cols,(const PetscScalar**)&vals);CHKERRQ(ierr);
for (k=0; k<j; k++) colsg[k] = indexg[cols[k]];
ierr = MatSetValues(temp_local_mat,1,&indexg[i],j,colsg,vals,ADD_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(*usedmat,i,&j,(const PetscInt**)&cols,(const PetscScalar**)&vals);CHKERRQ(ierr);
}
}
}
}
ierr = PetscFree(indexg);CHKERRQ(ierr);
ierr = PetscFree(colsg);CHKERRQ(ierr);
ierr = MatAssemblyBegin(temp_local_mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (temp_local_mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
#if DEBUG
{
Vec lvec,rvec;
PetscReal norm;
ierr = MatCreateVecs(temp_local_mat,&lvec,&rvec);CHKERRQ(ierr);
ierr = VecSet(lvec,1.0);CHKERRQ(ierr);
ierr = MatMult(temp_local_mat,lvec,rvec);CHKERRQ(ierr);
ierr = VecNorm(rvec,NORM_INFINITY,&norm);CHKERRQ(ierr);
printf("Test null space of local mat % 1.14e\n",norm);
ierr = VecDestroy(&lvec);CHKERRQ(ierr);
ierr = VecDestroy(&rvec);CHKERRQ(ierr);
}
#endif
*local_mat = temp_local_mat;
ierr = MatDestroy(&elem_mat_DBC);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Vec x,y,u,s1,s2;
Mat A,sA,sB;
PetscRandom rctx;
PetscReal r1,r2,rnorm,tol = PETSC_SQRT_MACHINE_EPSILON;
PetscScalar one=1.0, neg_one=-1.0, value[3], four=4.0,alpha=0.1;
PetscInt n,col[3],n1,block,row,i,j,i2,j2,Ii,J,rstart,rend,bs=1,mbs=16,d_nz=3,o_nz=3,prob=2;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscBool flg;
MatType type;
ierr = PetscInitialize(&argc,&args,(char*)0,help);
if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-mbs",&mbs,NULL);
CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);
CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);
CHKERRQ(ierr);
n = mbs*bs;
/* Assemble MPISBAIJ matrix sA */
ierr = MatCreate(PETSC_COMM_WORLD,&sA);
CHKERRQ(ierr);
ierr = MatSetSizes(sA,PETSC_DECIDE,PETSC_DECIDE,n,n);
CHKERRQ(ierr);
ierr = MatSetType(sA,MATSBAIJ);
CHKERRQ(ierr);
ierr = MatSetFromOptions(sA);
CHKERRQ(ierr);
ierr = MatGetType(sA,&type);
CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(sA,bs,d_nz,NULL,o_nz,NULL);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(sA,bs,d_nz,NULL);
CHKERRQ(ierr);
ierr = MatSetOption(sA,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);
CHKERRQ(ierr);
if (bs == 1) {
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0;
value[1] = 2.0;
value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = n - 1;
col[0]=0;
col[1] = n - 2;
col[2] = n - 1;
value[0]= 0.1;
value[1]=-1;
value[2]=2;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0;
col[0] = 0;
col[1] = 1;
col[2]=n-1;
value[0] = 2.0;
value[1] = -1.0;
value[2]=0.1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (int) PetscSqrtReal((PetscReal)n);
if (n1*n1 != n) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"n must be a perfect square of n1");
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
ierr = MatSetValues(sA,1,&Ii,1,&Ii,&four,INSERT_VALUES);
CHKERRQ(ierr);
}
}
}
/* end of if (bs == 1) */
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0;
value[1] = 4.0;
value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = bs - 1+block*bs;
col[0] = bs - 2+block*bs;
col[1] = bs - 1+block*bs;
value[0]=-1.0;
value[1]=4.0;
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0+block*bs;
col[0] = 0+block*bs;
col[1] = 1+block*bs;
value[0]=4.0;
value[1] = -1.0;
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(sA,1,&i,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
col[0]=i;
row=i+bs;
ierr = MatSetValues(sA,1,&row,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(sA,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(sA,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
/* Test MatView() */
ierr = MatCreateBAIJ(PETSC_COMM_WORLD,bs,PETSC_DECIDE,PETSC_DECIDE,n,n,d_nz,NULL,o_nz,NULL,&A);
CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);
CHKERRQ(ierr);
if (bs == 1) {
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0;
value[1] = 2.0;
value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = n - 1;
col[0]=0;
col[1] = n - 2;
col[2] = n - 1;
value[0]= 0.1;
value[1]=-1;
value[2]=2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0;
col[0] = 0;
col[1] = 1;
col[2]=n-1;
value[0] = 2.0;
value[1] = -1.0;
value[2]=0.1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (int) PetscSqrtReal((PetscReal)n);
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
ierr = MatSetValues(A,1,&Ii,1,&Ii,&four,INSERT_VALUES);
CHKERRQ(ierr);
}
}
}
/* end of if (bs == 1) */
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0;
value[1] = 4.0;
value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = bs - 1+block*bs;
col[0] = bs - 2+block*bs;
col[1] = bs - 1+block*bs;
value[0]=-1.0;
value[1]=4.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0+block*bs;
col[0] = 0+block*bs;
col[1] = 1+block*bs;
value[0]=4.0;
value[1] = -1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(A,1,&i,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
col[0]=i;
row=i+bs;
ierr = MatSetValues(A,1,&row,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
/* Test MatGetSize(), MatGetLocalSize() */
ierr = MatGetSize(sA, &i,&j);
CHKERRQ(ierr);
ierr = MatGetSize(A, &i2,&j2);
CHKERRQ(ierr);
i -= i2;
j -= j2;
if (i || j) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatGetSize()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatGetLocalSize(sA, &i,&j);
CHKERRQ(ierr);
ierr = MatGetLocalSize(A, &i2,&j2);
CHKERRQ(ierr);
i2 -= i;
j2 -= j;
if (i2 || j2) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatGetLocalSize()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* vectors */
/*--------------------*/
/* i is obtained from MatGetLocalSize() */
ierr = VecCreate(PETSC_COMM_WORLD,&x);
CHKERRQ(ierr);
ierr = VecSetSizes(x,i,PETSC_DECIDE);
CHKERRQ(ierr);
ierr = VecSetFromOptions(x);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&s1);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&s2);
CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);
CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);
CHKERRQ(ierr);
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = VecSet(u,one);
CHKERRQ(ierr);
/* Test MatNorm() */
ierr = MatNorm(A,NORM_FROBENIUS,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_FROBENIUS,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_FROBENIUS(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_INFINITY,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_INFINITY,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: MatNorm_INFINITY(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_1,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_1,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: MatNorm_1(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
/* Test MatGetOwnershipRange() */
ierr = MatGetOwnershipRange(sA,&rstart,&rend);
CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&i2,&j2);
CHKERRQ(ierr);
i2 -= rstart;
j2 -= rend;
if (i2 || j2) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MaGetOwnershipRange()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* Test MatDiagonalScale() */
ierr = MatDiagonalScale(A,x,x);
CHKERRQ(ierr);
ierr = MatDiagonalScale(sA,x,x);
CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDiagonalScale");
/* Test MatGetDiagonal(), MatScale() */
ierr = MatGetDiagonal(A,s1);
CHKERRQ(ierr);
ierr = MatGetDiagonal(sA,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDiagonalScale() or MatGetDiagonal(), r1=%g \n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatScale(A,alpha);
CHKERRQ(ierr);
ierr = MatScale(sA,alpha);
CHKERRQ(ierr);
/* Test MatGetRowMaxAbs() */
ierr = MatGetRowMaxAbs(A,s1,NULL);
CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(sA,s2,NULL);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetRowMaxAbs() \n");
CHKERRQ(ierr);
}
/* Test MatMult(), MatMultAdd() */
ierr = MatMultEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMult() or MatScale()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatMultAddEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMultAdd()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* Test MatMultTranspose(), MatMultTransposeAdd() */
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = MatMultTranspose(A,x,s1);
CHKERRQ(ierr);
ierr = MatMultTranspose(sA,x,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMult() or MatScale(), err=%g\n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
}
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = VecSetRandom(y,rctx);
CHKERRQ(ierr);
ierr = MatMultTransposeAdd(A,x,y,s1);
CHKERRQ(ierr);
ierr = MatMultTransposeAdd(sA,x,y,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMultAdd(), err=%g \n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
}
/* Test MatDuplicate() */
ierr = MatDuplicate(sA,MAT_COPY_VALUES,&sB);
CHKERRQ(ierr);
ierr = MatEqual(sA,sB,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD," Error in MatDuplicate(), sA != sB \n");
CHKERRQ(ierr);
CHKERRQ(ierr);
}
ierr = MatMultEqual(sA,sB,5,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDuplicate() or MatMult()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatMultAddEqual(sA,sB,5,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDuplicate() or MatMultAdd(()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatDestroy(&sB);
CHKERRQ(ierr);
ierr = VecDestroy(&u);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&y);
CHKERRQ(ierr);
ierr = VecDestroy(&s1);
CHKERRQ(ierr);
ierr = VecDestroy(&s2);
CHKERRQ(ierr);
ierr = MatDestroy(&sA);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);
CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode MatGetSubMatrix_BlockMat(Mat A,IS isrow,IS iscol,MatReuse scall,Mat *B)
{
Mat_BlockMat *a = (Mat_BlockMat*)A->data;
Mat_SeqAIJ *c;
PetscErrorCode ierr;
PetscInt i,k,first,step,lensi,nrows,ncols;
PetscInt *j_new,*i_new,*aj = a->j,*ailen = a->ilen;
PetscScalar *a_new;
Mat C,*aa = a->a;
PetscBool stride,equal;
PetscFunctionBegin;
ierr = ISEqual(isrow,iscol,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Only for idential column and row indices");
ierr = PetscObjectTypeCompare((PetscObject)iscol,ISSTRIDE,&stride);CHKERRQ(ierr);
if (!stride) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Only for stride indices");
ierr = ISStrideGetInfo(iscol,&first,&step);CHKERRQ(ierr);
if (step != A->rmap->bs) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Can only select one entry from each block");
ierr = ISGetLocalSize(isrow,&nrows);CHKERRQ(ierr);
ncols = nrows;
/* create submatrix */
if (scall == MAT_REUSE_MATRIX) {
PetscInt n_cols,n_rows;
C = *B;
ierr = MatGetSize(C,&n_rows,&n_cols);CHKERRQ(ierr);
if (n_rows != nrows || n_cols != ncols) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Reused submatrix wrong size");
ierr = MatZeroEntries(C);CHKERRQ(ierr);
} else {
ierr = MatCreate(PetscObjectComm((PetscObject)A),&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,nrows,ncols,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
if (A->symmetric) {
ierr = MatSetType(C,MATSEQSBAIJ);CHKERRQ(ierr);
} else {
ierr = MatSetType(C,MATSEQAIJ);CHKERRQ(ierr);
}
ierr = MatSeqAIJSetPreallocation(C,0,ailen);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(C,1,0,ailen);CHKERRQ(ierr);
}
c = (Mat_SeqAIJ*)C->data;
/* loop over rows inserting into submatrix */
a_new = c->a;
j_new = c->j;
i_new = c->i;
for (i=0; i<nrows; i++) {
lensi = ailen[i];
for (k=0; k<lensi; k++) {
*j_new++ = *aj++;
ierr = MatGetValue(*aa++,first,first,a_new++);CHKERRQ(ierr);
}
i_new[i+1] = i_new[i] + lensi;
c->ilen[i] = lensi;
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
*B = C;
PetscFunctionReturn(0);
}
EXTERN_C_END
/*MC
MATSOLVERCHOLMOD = "cholmod" - A matrix type providing direct solvers (Cholesky) for sequential matrices
via the external package CHOLMOD.
./configure --download-cholmod to install PETSc to use CHOLMOD
Consult CHOLMOD documentation for more information about the Common parameters
which correspond to the options database keys below.
Options Database Keys:
+ -mat_cholmod_dbound <0> - Minimum absolute value of diagonal entries of D (None)
. -mat_cholmod_grow0 <1.2> - Global growth ratio when factors are modified (None)
. -mat_cholmod_grow1 <1.2> - Column growth ratio when factors are modified (None)
. -mat_cholmod_grow2 <5> - Affine column growth constant when factors are modified (None)
. -mat_cholmod_maxrank <8> - Max rank of update, larger values are faster but use more memory [2,4,8] (None)
. -mat_cholmod_factor <AUTO> - (choose one of) SIMPLICIAL AUTO SUPERNODAL
. -mat_cholmod_supernodal_switch <40> - flop/nnz_L threshold for switching to supernodal factorization (None)
. -mat_cholmod_final_asis <TRUE> - Leave factors "as is" (None)
. -mat_cholmod_final_pack <TRUE> - Pack the columns when finished (use FALSE if the factors will be updated later) (None)
. -mat_cholmod_zrelax <0.8> - 3 real supernodal relaxed amalgamation parameters (None)
. -mat_cholmod_nrelax <4> - 3 size_t supernodal relaxed amalgamation parameters (None)
. -mat_cholmod_prefer_upper <TRUE> - Work with upper triangular form (faster when using fill-reducing ordering, slower in natural ordering) (None)
- -mat_cholmod_print <3> - Verbosity level (None)
Level: beginner
.seealso: PCCHOLESKY, PCFactorSetMatSolverPackage(), MatSolverPackage
M*/
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "MatGetFactor_seqsbaij_cholmod"
PetscErrorCode MatGetFactor_seqsbaij_cholmod(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
Mat_CHOLMOD *chol;
PetscErrorCode ierr;
PetscInt m=A->rmap->n,n=A->cmap->n,bs;
PetscFunctionBegin;
if (ftype != MAT_FACTOR_CHOLESKY) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"CHOLMOD cannot do %s factorization with SBAIJ, only %s",
MatFactorTypes[ftype],MatFactorTypes[MAT_FACTOR_CHOLESKY]);
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
if (bs != 1) SETERRQ1(((PetscObject)A)->comm,PETSC_ERR_SUP,"CHOLMOD only supports block size=1, given %D",bs);
/* Create the factorization matrix F */
ierr = MatCreate(((PetscObject)A)->comm,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,PETSC_DECIDE,PETSC_DECIDE,m,n);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(B,1,0,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscNewLog(B,Mat_CHOLMOD,&chol);CHKERRQ(ierr);
chol->Wrap = MatWrapCholmod_seqsbaij;
chol->Destroy = MatDestroy_SeqSBAIJ;
B->spptr = chol;
B->ops->view = MatView_CHOLMOD;
B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_CHOLMOD;
B->ops->destroy = MatDestroy_CHOLMOD;
ierr = PetscObjectComposeFunctionDynamic((PetscObject)B,"MatFactorGetSolverPackage_C","MatFactorGetSolverPackage_seqsbaij_cholmod",MatFactorGetSolverPackage_seqsbaij_cholmod);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_CHOLESKY;
B->assembled = PETSC_TRUE; /* required by -ksp_view */
B->preallocated = PETSC_TRUE;
ierr = CholmodStart(B);CHKERRQ(ierr);
*F = B;
PetscFunctionReturn(0);
}
int
PetscSparseMtrx :: buildInternalStructure(EngngModel *eModel, int di, EquationID ut, const UnknownNumberingScheme &s)
{
IntArray loc;
Domain *domain = eModel->giveDomain(di);
int nelem;
if ( mtrx ) {
MatDestroy(&mtrx);
}
if ( this->kspInit ) {
KSPDestroy(&ksp);
this->kspInit = false; // force ksp to be initialized
}
this->ut = ut;
this->emodel = eModel;
this->di = di;
#ifdef __PARALLEL_MODE
if ( eModel->isParallel() ) {
int rank;
PetscNatural2GlobalOrdering *n2g;
PetscNatural2LocalOrdering *n2l;
rank = eModel->giveRank();
n2g = eModel->givePetscContext(di, ut)->giveN2Gmap();
n2l = eModel->givePetscContext(di, ut)->giveN2Lmap();
n2l->init(eModel, ut, di);
n2g->init(eModel, ut, di);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscSparseMtrx:: buildInternalStructure", "", rank);
#endif
leqs = n2g->giveNumberOfLocalEqs();
geqs = n2g->giveNumberOfGlobalEqs();
//printf("%d, leqs = %d, geqs = %d\n", this->emodel->giveRank(), leqs, geqs);
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_INFO( "[%d]PetscSparseMtrx:: buildInternalStructure: l_eqs = %d, g_eqs = %d, n_eqs = %d\n", rank, leqs, geqs, eModel->giveNumberOfEquations(ut) );
#endif
// determine nonzero structure of a "local (maintained)" part of matrix, and the off-diagonal part
int i, ii, j, jj, n;
Element *elem;
// allocation map
std :: vector< std :: set< int > >d_rows(leqs); // diagonal sub-matrix allocation
std :: vector< std :: set< int > >o_rows(leqs); // off-diagonal allocation
IntArray d_nnz(leqs), o_nnz(leqs), lloc, gloc;
//fprintf (stderr,"[%d] n2l map: ",rank);
//for (n=1; n<=n2l.giveN2Lmap()->giveSize(); n++) fprintf (stderr, "%d ", n2l.giveN2Lmap()->at(n));
nelem = domain->giveNumberOfElements();
for ( n = 1; n <= nelem; n++ ) {
//fprintf (stderr, "(elem %d) ", n);
elem = domain->giveElement(n);
elem->giveLocationArray(loc, ut, s);
n2l->map2New(lloc, loc, 0); // translate natural->local numbering (remark, 1-based indexing)
n2g->map2New(gloc, loc, 0); // translate natural->global numbering (remark, 0-based indexing)
// See the petsc manual for details on how this allocation is constructed.
for ( i = 1; i <= lloc.giveSize(); i++ ) {
if ( ( ii = lloc.at(i) ) ) {
for ( j = 1; j <= lloc.giveSize(); j++ ) {
if ( ( jj = gloc.at(j) ) >= 0 ) { // if negative, then it is prescribed
if ( lloc.at(j) ) { // if true, then its the local part (the diagonal block matrix)
d_rows [ ii - 1 ].insert(jj);
} else { // Otherwise it must be off-diagonal
o_rows [ ii - 1 ].insert(jj);
}
}
}
}
}
//fprintf (stderr, "\n");
}
// Diagonal must always be allocated; this code ensures that for every local line, it adds the global column number
IntArray *n2gmap = n2g->giveN2Gmap();
IntArray *n2lmap = n2l->giveN2Lmap();
for ( int n = 1; n <= n2lmap->giveSize(); ++n ) {
if ( n2lmap->at(n) ) {
d_rows [ n2lmap->at(n)-1 ].insert( n2gmap->at(n) );
}
}
for ( i = 0; i < leqs; i++ ) {
d_nnz(i) = d_rows [ i ].size();
o_nnz(i) = o_rows [ i ].size();
}
//fprintf (stderr,"\n[%d]PetscSparseMtrx: Profile ...",rank);
//for (i=0; i<leqs; i++) fprintf(stderr, "%d ", d_nnz(i));
//fprintf (stderr,"\n[%d]PetscSparseMtrx: Creating MPIAIJ Matrix ...\n",rank);
// create PETSc mat
MatCreate(PETSC_COMM_WORLD, & mtrx);
MatSetSizes(mtrx, leqs, leqs, geqs, geqs);
MatSetType(mtrx, MATMPIAIJ);
MatSetFromOptions(mtrx);
MatSetUp(mtrx);
MatMPIAIJSetPreallocation(mtrx, 0, d_nnz.givePointer(), 0, o_nnz.givePointer());
//MatMPIBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz.givePointer(), onz, onnz );
//MatMPISBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz_sym.givePointer(), onz, onnz );
MatSetOption(mtrx, MAT_ROW_ORIENTED, PETSC_FALSE); // To allow the insertion of values using MatSetValues in column major order
MatSetOption(mtrx, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscSparseMtrx:: buildInternalStructure", "done", rank);
#endif
} else {
#endif
leqs = geqs = eModel->giveNumberOfEquations(ut);
int i, ii, j, jj, n;
Element *elem;
// allocation map
std :: vector< std :: set< int > >rows(leqs);
std :: vector< std :: set< int > >rows_sym(leqs);
nelem = domain->giveNumberOfElements();
for ( n = 1; n <= nelem; n++ ) {
elem = domain->giveElement(n);
elem->giveLocationArray(loc, ut, s);
for ( i = 1; i <= loc.giveSize(); i++ ) {
if ( ( ii = loc.at(i) ) ) {
for ( j = 1; j <= loc.giveSize(); j++ ) {
jj = loc.at(j);
if ( jj ) {
rows [ ii - 1 ].insert(jj - 1);
if ( jj >= ii ) {
rows_sym [ ii - 1 ].insert(jj - 1);
}
}
}
}
}
}
// Structure from active boundary conditions.
AList<IntArray> locs, temp;
for ( n = 1; n <= domain->giveNumberOfBoundaryConditions(); n++ ) {
ActiveBoundaryCondition *activebc = dynamic_cast<ActiveBoundaryCondition*>(domain->giveBc(n));
if (activebc) {
///@todo Deal with the CharType here.
activebc->giveLocationArrays(locs, temp, ut, TangentStiffnessMatrix, s, s, domain);
for (int k = 1; k < locs.giveSize(); k++) {
IntArray *kloc = locs.at(k);
for ( i = 1; i <= kloc->giveSize(); i++ ) {
if ( ( ii = kloc->at(i) ) ) {
for ( j = 1; j <= kloc->giveSize(); j++ ) {
jj = kloc->at(j);
if ( jj ) {
rows [ ii - 1 ].insert(jj - 1);
if ( jj >= ii ) {
rows_sym [ ii - 1 ].insert(jj - 1);
}
}
}
}
}
}
}
}
IntArray d_nnz(leqs);
IntArray d_nnz_sym(leqs);
for ( i = 0; i < leqs; i++ ) {
d_nnz(i) = rows [ i ].size();
d_nnz_sym(i) = rows_sym [ i ].size();
}
MatCreate(PETSC_COMM_SELF, & mtrx);
MatSetSizes(mtrx, leqs, leqs, geqs, geqs);
MatSetType(mtrx, MATSEQAIJ);
//MatSetType(mtrx, MATSBAIJ);
//MatSetType(mtrx, MATDENSE);
MatSetFromOptions(mtrx);
MatSetUp(mtrx);
MatSeqAIJSetPreallocation( mtrx, 0, d_nnz.givePointer() );
MatSeqBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz.givePointer() );
MatSeqSBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz_sym.givePointer() );
MatSetOption(mtrx, MAT_ROW_ORIENTED, PETSC_FALSE); // To allow the insertion of values using MatSetValues in column major order
MatSetOption(mtrx, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
#ifdef __PARALLEL_MODE
}
#endif
nRows = nColumns = geqs;
this->newValues = true;
return true;
}