/*
MatConvert_Basic - Converts from any input format to another format. For
parallel formats, the new matrix distribution is determined by PETSc.
Does not do preallocation so in general will be slow
*/
PETSC_INTERN PetscErrorCode MatConvert_Basic(Mat mat, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat M;
const PetscScalar *vwork;
PetscErrorCode ierr;
PetscInt nz,i,m,n,rstart,rend,lm,ln;
const PetscInt *cwork;
PetscBool isSBAIJ;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)mat,MATSEQSBAIJ,&isSBAIJ);CHKERRQ(ierr);
if (!isSBAIJ) {
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPISBAIJ,&isSBAIJ);CHKERRQ(ierr);
}
if (isSBAIJ) SETERRQ(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"Cannot convert from SBAIJ matrix since cannot obtain entire rows of matrix");
ierr = MatGetSize(mat,&m,&n);CHKERRQ(ierr);
ierr = MatGetLocalSize(mat,&lm,&ln);CHKERRQ(ierr);
if (ln == n) ln = PETSC_DECIDE; /* try to preserve column ownership */
ierr = MatCreate(PetscObjectComm((PetscObject)mat),&M);CHKERRQ(ierr);
ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr);
ierr = MatSetBlockSizesFromMats(M,mat,mat);CHKERRQ(ierr);
ierr = MatSetType(M,newtype);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(M,NULL);CHKERRQ(ierr);
ierr = MatMPIDenseSetPreallocation(M,NULL);CHKERRQ(ierr);
ierr = MatSetUp(M);CHKERRQ(ierr);
ierr = MatSetOption(M,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatSetOption(M,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATSEQSBAIJ,&isSBAIJ);CHKERRQ(ierr);
if (!isSBAIJ) {
ierr = PetscObjectTypeCompare((PetscObject)M,MATMPISBAIJ,&isSBAIJ);CHKERRQ(ierr);
}
if (isSBAIJ) {
ierr = MatSetOption(M,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
}
ierr = MatGetOwnershipRange(mat,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatGetRow(mat,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
ierr = MatSetValues(M,1,&i,nz,cwork,vwork,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(mat,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_INPLACE_MATRIX) {
ierr = MatHeaderReplace(mat,&M);CHKERRQ(ierr);
} else {
*newmat = M;
}
PetscFunctionReturn(0);
}
PETSC_INTERN PetscErrorCode MatConvert_Shell(Mat oldmat, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat mat;
Vec in,out;
PetscErrorCode ierr;
PetscInt i,M,m,*rows,start,end;
MPI_Comm comm;
PetscScalar *array,zero = 0.0,one = 1.0;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)oldmat,&comm);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(oldmat,&start,&end);CHKERRQ(ierr);
ierr = VecCreateMPI(comm,end-start,PETSC_DECIDE,&in);CHKERRQ(ierr);
ierr = VecDuplicate(in,&out);CHKERRQ(ierr);
ierr = VecGetSize(in,&M);CHKERRQ(ierr);
ierr = VecGetLocalSize(in,&m);CHKERRQ(ierr);
ierr = PetscMalloc1(m+1,&rows);CHKERRQ(ierr);
for (i=0; i<m; i++) rows[i] = start + i;
ierr = MatCreate(comm,&mat);CHKERRQ(ierr);
ierr = MatSetSizes(mat,m,M,M,M);CHKERRQ(ierr);
ierr = MatSetType(mat,newtype);CHKERRQ(ierr);
ierr = MatSetBlockSizesFromMats(mat,oldmat,oldmat);CHKERRQ(ierr);
ierr = MatSetUp(mat);CHKERRQ(ierr);
for (i=0; i<M; i++) {
ierr = VecSet(in,zero);CHKERRQ(ierr);
ierr = VecSetValues(in,1,&i,&one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(in);CHKERRQ(ierr);
ierr = VecAssemblyEnd(in);CHKERRQ(ierr);
ierr = MatMult(oldmat,in,out);CHKERRQ(ierr);
ierr = VecGetArray(out,&array);CHKERRQ(ierr);
ierr = MatSetValues(mat,m,rows,1,&i,array,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecRestoreArray(out,&array);CHKERRQ(ierr);
}
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = VecDestroy(&in);CHKERRQ(ierr);
ierr = VecDestroy(&out);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_INPLACE_MATRIX) {
ierr = MatHeaderReplace(oldmat,&mat);CHKERRQ(ierr);
} else {
*newmat = mat;
}
PetscFunctionReturn(0);
}
/*
Takes the local part of an already assembled MPIAIJ matrix
and disassembles it. This is to allow new nonzeros into the matrix
that require more communication in the matrix vector multiply.
Thus certain data-structures must be rebuilt.
Kind of slow! But that's what application programmers get when
they are sloppy.
*/
PetscErrorCode MatDisAssemble_MPIAIJ(Mat A)
{
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)A->data;
Mat B = aij->B,Bnew;
Mat_SeqAIJ *Baij = (Mat_SeqAIJ*)B->data;
PetscErrorCode ierr;
PetscInt i,j,m = B->rmap->n,n = A->cmap->N,col,ct = 0,*garray = aij->garray,*nz,ec;
PetscScalar v;
PetscFunctionBegin;
/* free stuff related to matrix-vec multiply */
ierr = VecGetSize(aij->lvec,&ec);CHKERRQ(ierr); /* needed for PetscLogObjectMemory below */
ierr = VecDestroy(&aij->lvec);CHKERRQ(ierr);
if (aij->colmap) {
#if defined(PETSC_USE_CTABLE)
ierr = PetscTableDestroy(&aij->colmap);CHKERRQ(ierr);
#else
ierr = PetscFree(aij->colmap);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)A,-aij->B->cmap->n*sizeof(PetscInt));CHKERRQ(ierr);
#endif
}
/* make sure that B is assembled so we can access its values */
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* invent new B and copy stuff over */
ierr = PetscMalloc1(m+1,&nz);CHKERRQ(ierr);
for (i=0; i<m; i++) {
nz[i] = Baij->i[i+1] - Baij->i[i];
}
ierr = MatCreate(PETSC_COMM_SELF,&Bnew);CHKERRQ(ierr);
ierr = MatSetSizes(Bnew,m,n,m,n);CHKERRQ(ierr);
ierr = MatSetBlockSizesFromMats(Bnew,A,A);CHKERRQ(ierr);
ierr = MatSetType(Bnew,((PetscObject)B)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(Bnew,0,nz);CHKERRQ(ierr);
if (Baij->nonew >= 0) { /* Inherit insertion error options (if positive). */
((Mat_SeqAIJ*)Bnew->data)->nonew = Baij->nonew;
}
/*
Ensure that B's nonzerostate is monotonically increasing.
Or should this follow the MatSetValues() loop to preserve B's nonzerstate across a MatDisAssemble() call?
*/
Bnew->nonzerostate = B->nonzerostate;
ierr = PetscFree(nz);CHKERRQ(ierr);
for (i=0; i<m; i++) {
for (j=Baij->i[i]; j<Baij->i[i+1]; j++) {
col = garray[Baij->j[ct]];
v = Baij->a[ct++];
ierr = MatSetValues(Bnew,1,&i,1,&col,&v,B->insertmode);CHKERRQ(ierr);
}
}
ierr = PetscFree(aij->garray);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)A,-ec*sizeof(PetscInt));CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)A,(PetscObject)Bnew);CHKERRQ(ierr);
aij->B = Bnew;
A->was_assembled = PETSC_FALSE;
PetscFunctionReturn(0);
}
/*
MatConvert_Basic - Converts from any input format to another format. For
parallel formats, the new matrix distribution is determined by PETSc.
Does not do preallocation so in general will be slow
*/
PetscErrorCode MatConvert_Basic(Mat mat, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat M;
const PetscScalar *vwork;
PetscErrorCode ierr;
PetscInt i,j,nz,m,n,rstart,rend,lm,ln,prbs,pcbs,cstart,cend,*dnz,*onz;
const PetscInt *cwork;
PetscBool isseqsbaij,ismpisbaij,isseqbaij,ismpibaij,isseqdense,ismpidense;
PetscFunctionBegin;
ierr = MatGetSize(mat,&m,&n);CHKERRQ(ierr);
ierr = MatGetLocalSize(mat,&lm,&ln);CHKERRQ(ierr);
if (ln == n) ln = PETSC_DECIDE; /* try to preserve column ownership */
ierr = MatCreate(PetscObjectComm((PetscObject)mat),&M);CHKERRQ(ierr);
ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr);
ierr = MatSetBlockSizesFromMats(M,mat,mat);CHKERRQ(ierr);
ierr = MatSetType(M,newtype);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(mat,&rstart,&rend);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATSEQSBAIJ,&isseqsbaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATMPISBAIJ,&ismpisbaij);CHKERRQ(ierr);
if (isseqsbaij || ismpisbaij) {ierr = MatSetOption(M,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);}
ierr = PetscObjectTypeCompare((PetscObject)M,MATSEQBAIJ,&isseqbaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATMPIBAIJ,&ismpibaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATSEQDENSE,&isseqdense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATMPIDENSE,&ismpidense);CHKERRQ(ierr);
if (isseqdense) {
ierr = MatSeqDenseSetPreallocation(M,NULL);CHKERRQ(ierr);
} else if (ismpidense) {
ierr = MatMPIDenseSetPreallocation(M,NULL);CHKERRQ(ierr);
} else {
/* Preallocation block sizes. (S)BAIJ matrices will have one index per block. */
prbs = (isseqbaij || ismpibaij || isseqsbaij || ismpisbaij) ? PetscAbs(M->rmap->bs) : 1;
pcbs = (isseqbaij || ismpibaij || isseqsbaij || ismpisbaij) ? PetscAbs(M->cmap->bs) : 1;
ierr = PetscMalloc2(lm/prbs,&dnz,lm/prbs,&onz);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(mat,&cstart,&cend);CHKERRQ(ierr);
for (i=0; i<lm; i+=prbs) {
ierr = MatGetRow(mat,rstart+i,&nz,&cwork,NULL);CHKERRQ(ierr);
dnz[i] = 0;
onz[i] = 0;
for (j=0; j<nz; j+=pcbs) {
if ((isseqsbaij || ismpisbaij) && cwork[j] < rstart+i) continue;
if (cstart <= cwork[j] && cwork[j] < cend) dnz[i]++;
else onz[i]++;
}
ierr = MatRestoreRow(mat,rstart+i,&nz,&cwork,NULL);CHKERRQ(ierr);
}
ierr = MatXAIJSetPreallocation(M,PETSC_DECIDE,dnz,onz,dnz,onz);CHKERRQ(ierr);
ierr = PetscFree2(dnz,onz);CHKERRQ(ierr);
}
for (i=rstart; i<rend; i++) {
ierr = MatGetRow(mat,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
ierr = MatSetValues(M,1,&i,nz,cwork,vwork,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(mat,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_REUSE_MATRIX) {
ierr = MatHeaderReplace(mat,M);CHKERRQ(ierr);
} else {
*newmat = M;
}
PetscFunctionReturn(0);
}
PetscErrorCode MatGetMultiProcBlock_MPIAIJ(Mat mat, MPI_Comm subComm, MatReuse scall,Mat *subMat)
{
PetscErrorCode ierr;
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)mat->data;
Mat_SeqAIJ *aijB = (Mat_SeqAIJ*)aij->B->data;
PetscMPIInt commRank,subCommSize,subCommRank;
PetscMPIInt *commRankMap,subRank,rank,commsize;
PetscInt *garrayCMap,col,i,j,*nnz,newRow,newCol;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&commsize);CHKERRQ(ierr);
ierr = MPI_Comm_size(subComm,&subCommSize);CHKERRQ(ierr);
/* create subMat object with the relavent layout */
if (scall == MAT_INITIAL_MATRIX) {
ierr = MatCreate(subComm,subMat);CHKERRQ(ierr);
ierr = MatSetType(*subMat,MATMPIAIJ);CHKERRQ(ierr);
ierr = MatSetSizes(*subMat,mat->rmap->n,mat->cmap->n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetBlockSizesFromMats(*subMat,mat,mat);CHKERRQ(ierr);
/* need to setup rmap and cmap before Preallocation */
ierr = PetscLayoutSetUp((*subMat)->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp((*subMat)->cmap);CHKERRQ(ierr);
}
/* create a map of comm_rank from subComm to comm - should commRankMap and garrayCMap be kept for reused? */
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)mat),&commRank);CHKERRQ(ierr);
ierr = MPI_Comm_rank(subComm,&subCommRank);CHKERRQ(ierr);
ierr = PetscMalloc1(subCommSize,&commRankMap);CHKERRQ(ierr);
ierr = MPI_Allgather(&commRank,1,MPI_INT,commRankMap,1,MPI_INT,subComm);CHKERRQ(ierr);
/* Traverse garray and identify column indices [of offdiag mat] that
should be discarded. For the ones not discarded, store the newCol+1
value in garrayCMap */
ierr = PetscCalloc1(aij->B->cmap->n,&garrayCMap);CHKERRQ(ierr);
for (i=0; i<aij->B->cmap->n; i++) {
col = aij->garray[i];
for (subRank=0; subRank<subCommSize; subRank++) {
rank = commRankMap[subRank];
if ((col >= mat->cmap->range[rank]) && (col < mat->cmap->range[rank+1])) {
garrayCMap[i] = (*subMat)->cmap->range[subRank] + col - mat->cmap->range[rank]+1;
break;
}
}
}
if (scall == MAT_INITIAL_MATRIX) {
/* Now compute preallocation for the offdiag mat */
ierr = PetscCalloc1(aij->B->rmap->n,&nnz);CHKERRQ(ierr);
for (i=0; i<aij->B->rmap->n; i++) {
for (j=aijB->i[i]; j<aijB->i[i+1]; j++) {
if (garrayCMap[aijB->j[j]]) nnz[i]++;
}
}
ierr = MatMPIAIJSetPreallocation(*(subMat),0,NULL,0,nnz);CHKERRQ(ierr);
/* reuse diag block with the new submat */
ierr = MatDestroy(&((Mat_MPIAIJ*)((*subMat)->data))->A);CHKERRQ(ierr);
((Mat_MPIAIJ*)((*subMat)->data))->A = aij->A;
ierr = PetscObjectReference((PetscObject)aij->A);CHKERRQ(ierr);
} else if (((Mat_MPIAIJ*)(*subMat)->data)->A != aij->A) {
PetscObject obj = (PetscObject)((Mat_MPIAIJ*)((*subMat)->data))->A;
ierr = PetscObjectReference((PetscObject)obj);CHKERRQ(ierr);
((Mat_MPIAIJ*)((*subMat)->data))->A = aij->A;
ierr = PetscObjectReference((PetscObject)aij->A);CHKERRQ(ierr);
}
/* Now traverse aij->B and insert values into subMat */
for (i=0; i<aij->B->rmap->n; i++) {
newRow = (*subMat)->rmap->range[subCommRank] + i;
for (j=aijB->i[i]; j<aijB->i[i+1]; j++) {
newCol = garrayCMap[aijB->j[j]];
if (newCol) {
newCol--; /* remove the increment */
ierr = MatSetValues(*subMat,1,&newRow,1,&newCol,(aijB->a+j),INSERT_VALUES);CHKERRQ(ierr);
}
}
}
/* assemble the submat */
ierr = MatAssemblyBegin(*subMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*subMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* deallocate temporary data */
ierr = PetscFree(commRankMap);CHKERRQ(ierr);
ierr = PetscFree(garrayCMap);CHKERRQ(ierr);
if (scall == MAT_INITIAL_MATRIX) {
ierr = PetscFree(nnz);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@
MatCreateSubMatrix - Creates a composite matrix that acts as a submatrix
Collective on Mat
Input Parameters:
+ A - matrix that we will extract a submatrix of
. isrow - rows to be present in the submatrix
- iscol - columns to be present in the submatrix
Output Parameters:
. newmat - new matrix
Level: developer
Notes:
Most will use MatGetSubMatrix which provides a more efficient representation if it is available.
.seealso: MatGetSubMatrix(), MatSubMatrixUpdate()
@*/
PetscErrorCode MatCreateSubMatrix(Mat A,IS isrow,IS iscol,Mat *newmat)
{
Vec left,right;
PetscInt m,n;
Mat N;
Mat_SubMatrix *Na;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(A,MAT_CLASSID,1);
PetscValidHeaderSpecific(isrow,IS_CLASSID,2);
PetscValidHeaderSpecific(iscol,IS_CLASSID,3);
PetscValidPointer(newmat,4);
*newmat = 0;
ierr = MatCreate(PetscObjectComm((PetscObject)A),&N);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrow,&m);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscol,&n);CHKERRQ(ierr);
ierr = MatSetSizes(N,m,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = PetscObjectChangeTypeName((PetscObject)N,MATSUBMATRIX);CHKERRQ(ierr);
ierr = PetscNewLog(N,&Na);CHKERRQ(ierr);
N->data = (void*)Na;
ierr = PetscObjectReference((PetscObject)A);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)isrow);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)iscol);CHKERRQ(ierr);
Na->A = A;
Na->isrow = isrow;
Na->iscol = iscol;
Na->scale = 1.0;
N->ops->destroy = MatDestroy_SubMatrix;
N->ops->mult = MatMult_SubMatrix;
N->ops->multadd = MatMultAdd_SubMatrix;
N->ops->multtranspose = MatMultTranspose_SubMatrix;
N->ops->multtransposeadd = MatMultTransposeAdd_SubMatrix;
N->ops->scale = MatScale_SubMatrix;
N->ops->diagonalscale = MatDiagonalScale_SubMatrix;
ierr = MatSetBlockSizesFromMats(N,A,A);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(N->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(N->cmap);CHKERRQ(ierr);
ierr = MatGetVecs(A,&Na->rwork,&Na->lwork);CHKERRQ(ierr);
ierr = VecCreate(PetscObjectComm((PetscObject)isrow),&left);CHKERRQ(ierr);
ierr = VecCreate(PetscObjectComm((PetscObject)iscol),&right);CHKERRQ(ierr);
ierr = VecSetSizes(left,m,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetSizes(right,n,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetUp(left);CHKERRQ(ierr);
ierr = VecSetUp(right);CHKERRQ(ierr);
ierr = VecScatterCreate(Na->lwork,isrow,left,NULL,&Na->lrestrict);CHKERRQ(ierr);
ierr = VecScatterCreate(right,NULL,Na->rwork,iscol,&Na->rprolong);CHKERRQ(ierr);
ierr = VecDestroy(&left);CHKERRQ(ierr);
ierr = VecDestroy(&right);CHKERRQ(ierr);
N->assembled = PETSC_TRUE;
ierr = MatSetUp(N);CHKERRQ(ierr);
*newmat = N;
PetscFunctionReturn(0);
}
