void linearSystemPETSc<scalar>::allocate(int nbRows)
{
int commSize;
MPI_Comm_size(_comm, &commSize);
int blockSize = _getBlockSizeFromParameters();
clear();
_try(MatCreate(_comm, &_a));
_try(MatSetSizes(_a, blockSize * nbRows, blockSize * nbRows, PETSC_DETERMINE, PETSC_DETERMINE));
if (blockSize > 1) {
if (commSize > 1) {
_try(MatSetType(_a, MATMPIBAIJ));
}
else {
_try(MatSetType(_a, MATSEQBAIJ));
}
}
// override the default options with the ones from the option
// database (if any)
if (this->_parameters.count("petscOptions"))
_try(PetscOptionsInsertString(this->_parameters["petscOptions"].c_str()));
if (this->_parameters.count("petscPrefix"))
_try(MatAppendOptionsPrefix(_a, this->_parameters["petscPrefix"].c_str()));
_try(MatSetFromOptions(_a));
//since PETSc 3.3 GetOwnershipRange and MatGetSize cannot be called before MatXXXSetPreallocation
_localSize = nbRows;
#ifdef HAVE_MPI
if (commSize>1){
_localRowStart = 0;
if (Msg::GetCommRank() != 0) {
MPI_Status status;
MPI_Recv((void*)&_localRowStart, 1, MPI_INT, Msg::GetCommRank() - 1, 1, MPI_COMM_WORLD, &status);
}
_localRowEnd = _localRowStart + nbRows;
if (Msg::GetCommRank() != Msg::GetCommSize() - 1) {
MPI_Send((void*)&_localRowEnd, 1, MPI_INT, Msg::GetCommRank() + 1, 1, MPI_COMM_WORLD);
}
MPI_Allreduce((void*)&_localSize, (void*)&_globalSize, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
}
else{
_localRowStart = 0;
_localRowEnd = nbRows;
_globalSize = _localSize;
}
#else
_localRowStart = 0;
_localRowEnd = nbRows;
_globalSize = _localSize;
#endif
// preallocation option must be set after other options
_try(VecCreate(_comm, &_x));
_try(VecSetSizes(_x, blockSize * nbRows, PETSC_DETERMINE));
// override the default options with the ones from the option
// database (if any)
if (this->_parameters.count("petscPrefix"))
_try(VecAppendOptionsPrefix(_x, this->_parameters["petscPrefix"].c_str()));
_try(VecSetFromOptions(_x));
_try(VecDuplicate(_x, &_b));
_isAllocated = true;
_entriesPreAllocated = false;
}
PetscErrorCode MatSetLocalToGlobalMapping_IS(Mat A,ISLocalToGlobalMapping rmapping,ISLocalToGlobalMapping cmapping)
{
PetscErrorCode ierr;
PetscInt n,bs;
Mat_IS *is = (Mat_IS*)A->data;
IS from,to;
Vec global;
PetscFunctionBegin;
PetscCheckSameComm(A,1,rmapping,2);
if (rmapping != cmapping) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_INCOMP,"MATIS requires the row and column mappings to be identical");
if (is->mapping) { /* Currenly destroys the objects that will be created by this routine. Is there anything else that should be checked? */
ierr = ISLocalToGlobalMappingDestroy(&is->mapping);CHKERRQ(ierr);
ierr = VecDestroy(&is->x);CHKERRQ(ierr);
ierr = VecDestroy(&is->y);CHKERRQ(ierr);
ierr = VecScatterDestroy(&is->ctx);CHKERRQ(ierr);
ierr = MatDestroy(&is->A);CHKERRQ(ierr);
}
ierr = PetscObjectReference((PetscObject)rmapping);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&is->mapping);CHKERRQ(ierr);
is->mapping = rmapping;
/*
ierr = PetscLayoutSetISLocalToGlobalMapping(A->rmap,rmapping);CHKERRQ(ierr);
ierr = PetscLayoutSetISLocalToGlobalMapping(A->cmap,cmapping);CHKERRQ(ierr);
*/
/* Create the local matrix A */
ierr = ISLocalToGlobalMappingGetSize(rmapping,&n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetBlockSize(rmapping,&bs);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_SELF,&is->A);CHKERRQ(ierr);
if (bs > 1) {
ierr = MatSetType(is->A,MATSEQBAIJ);CHKERRQ(ierr);
} else {
ierr = MatSetType(is->A,MATSEQAIJ);CHKERRQ(ierr);
}
ierr = MatSetSizes(is->A,n,n,n,n);CHKERRQ(ierr);
ierr = MatSetBlockSize(is->A,bs);CHKERRQ(ierr);
ierr = MatSetOptionsPrefix(is->A,((PetscObject)A)->prefix);CHKERRQ(ierr);
ierr = MatAppendOptionsPrefix(is->A,"is_");CHKERRQ(ierr);
ierr = MatSetFromOptions(is->A);CHKERRQ(ierr);
/* Create the local work vectors */
ierr = VecCreate(PETSC_COMM_SELF,&is->x);CHKERRQ(ierr);
ierr = VecSetBlockSize(is->x,bs);CHKERRQ(ierr);
ierr = VecSetSizes(is->x,n,n);CHKERRQ(ierr);
ierr = VecSetOptionsPrefix(is->x,((PetscObject)A)->prefix);CHKERRQ(ierr);
ierr = VecAppendOptionsPrefix(is->x,"is_");CHKERRQ(ierr);
ierr = VecSetFromOptions(is->x);CHKERRQ(ierr);
ierr = VecDuplicate(is->x,&is->y);CHKERRQ(ierr);
/* setup the global to local scatter */
ierr = ISCreateStride(PETSC_COMM_SELF,n,0,1,&to);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApplyIS(rmapping,to,&from);CHKERRQ(ierr);
ierr = MatCreateVecs(A,&global,NULL);CHKERRQ(ierr);
ierr = VecScatterCreate(global,from,is->x,to,&is->ctx);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Developer Notes: This should be implemented with a MatCreate_SchurComplement() as that is the standard design for new Mat classes. */
PetscErrorCode MatGetSchurComplement_Basic(Mat mat,IS isrow0,IS iscol0,IS isrow1,IS iscol1,MatReuse mreuse,Mat *newmat,MatReuse preuse,Mat *newpmat)
{
PetscErrorCode ierr;
Mat A=0,Ap=0,B=0,C=0,D=0;
PetscFunctionBegin;
PetscValidHeaderSpecific(mat,MAT_CLASSID,1);
PetscValidHeaderSpecific(isrow0,IS_CLASSID,2);
PetscValidHeaderSpecific(iscol0,IS_CLASSID,3);
PetscValidHeaderSpecific(isrow1,IS_CLASSID,4);
PetscValidHeaderSpecific(iscol1,IS_CLASSID,5);
if (mreuse == MAT_REUSE_MATRIX) PetscValidHeaderSpecific(*newmat,MAT_CLASSID,7);
if (preuse == MAT_REUSE_MATRIX) PetscValidHeaderSpecific(*newpmat,MAT_CLASSID,9);
PetscValidType(mat,1);
if (mat->factortype) SETERRQ(((PetscObject)mat)->comm,PETSC_ERR_ARG_WRONGSTATE,"Not for factored matrix");
if (mreuse != MAT_IGNORE_MATRIX) {
/* Use MatSchurComplement */
if (mreuse == MAT_REUSE_MATRIX) {
ierr = MatSchurComplementGetSubmatrices(*newmat,&A,&Ap,&B,&C,&D);CHKERRQ(ierr);
if (!A || !Ap || !B || !C) SETERRQ(((PetscObject)mat)->comm,PETSC_ERR_ARG_WRONGSTATE,"Attempting to reuse matrix but Schur complement matrices unset");
if (A != Ap) SETERRQ(((PetscObject)mat)->comm,PETSC_ERR_ARG_WRONGSTATE,"Preconditioning matrix does not match operator");
ierr = MatDestroy(&Ap);CHKERRQ(ierr); /* get rid of extra reference */
}
ierr = MatGetSubMatrix(mat,isrow0,iscol0,mreuse,&A);CHKERRQ(ierr);
ierr = MatGetSubMatrix(mat,isrow0,iscol1,mreuse,&B);CHKERRQ(ierr);
ierr = MatGetSubMatrix(mat,isrow1,iscol0,mreuse,&C);CHKERRQ(ierr);
ierr = MatGetSubMatrix(mat,isrow1,iscol1,mreuse,&D);CHKERRQ(ierr);
switch (mreuse) {
case MAT_INITIAL_MATRIX:
ierr = MatCreateSchurComplement(A,A,B,C,D,newmat);CHKERRQ(ierr);
break;
case MAT_REUSE_MATRIX:
ierr = MatSchurComplementUpdate(*newmat,A,A,B,C,D,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
break;
default:
SETERRQ(((PetscObject)mat)->comm,PETSC_ERR_SUP,"Unrecognized value of mreuse");
}
}
if (preuse != MAT_IGNORE_MATRIX) {
/* Use the diagonal part of A to form D - C inv(diag(A)) B */
Mat Ad,AdB,S;
Vec diag;
PetscInt i,m,n,mstart,mend;
PetscScalar *x;
/* We could compose these with newpmat so that the matrices can be reused. */
if (!A) {ierr = MatGetSubMatrix(mat,isrow0,iscol0,MAT_INITIAL_MATRIX,&A);CHKERRQ(ierr);}
if (!B) {ierr = MatGetSubMatrix(mat,isrow0,iscol1,MAT_INITIAL_MATRIX,&B);CHKERRQ(ierr);}
if (!C) {ierr = MatGetSubMatrix(mat,isrow1,iscol0,MAT_INITIAL_MATRIX,&C);CHKERRQ(ierr);}
if (!D) {ierr = MatGetSubMatrix(mat,isrow1,iscol1,MAT_INITIAL_MATRIX,&D);CHKERRQ(ierr);}
ierr = MatGetVecs(A,&diag,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetDiagonal(A,diag);CHKERRQ(ierr);
ierr = VecReciprocal(diag);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
/* We need to compute S = D - C inv(diag(A)) B. For row-oriented formats, it is easy to scale the rows of B and
* for column-oriented formats the columns of C can be scaled. Would skip creating a silly diagonal matrix. */
ierr = MatCreate(((PetscObject)A)->comm,&Ad);CHKERRQ(ierr);
ierr = MatSetSizes(Ad,m,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetOptionsPrefix(Ad,((PetscObject)mat)->prefix);CHKERRQ(ierr);
ierr = MatAppendOptionsPrefix(Ad,"diag_");CHKERRQ(ierr);
ierr = MatSetFromOptions(Ad);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(Ad,1,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(Ad,1,PETSC_NULL,0,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Ad,&mstart,&mend);CHKERRQ(ierr);
ierr = VecGetArray(diag,&x);CHKERRQ(ierr);
for (i=mstart; i<mend; i++) {
ierr = MatSetValue(Ad,i,i,x[i-mstart],INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecRestoreArray(diag,&x);CHKERRQ(ierr);
ierr = MatAssemblyBegin(Ad,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Ad,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecDestroy(&diag);CHKERRQ(ierr);
ierr = MatMatMult(Ad,B,MAT_INITIAL_MATRIX,1,&AdB);CHKERRQ(ierr);
S = (preuse == MAT_REUSE_MATRIX) ? *newpmat : (Mat)0;
ierr = MatMatMult(C,AdB,preuse,PETSC_DEFAULT,&S);CHKERRQ(ierr);
ierr = MatAYPX(S,-1,D,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
*newpmat = S;
ierr = MatDestroy(&Ad);CHKERRQ(ierr);
ierr = MatDestroy(&AdB);CHKERRQ(ierr);
}
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&D);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatSetLocalToGlobalMapping_IS(Mat A,ISLocalToGlobalMapping rmapping,ISLocalToGlobalMapping cmapping)
{
PetscErrorCode ierr;
PetscInt nr,rbs,nc,cbs;
Mat_IS *is = (Mat_IS*)A->data;
IS from,to;
Vec cglobal,rglobal;
PetscFunctionBegin;
PetscCheckSameComm(A,1,rmapping,2);
PetscCheckSameComm(A,1,cmapping,3);
/* Destroy any previous data */
ierr = VecDestroy(&is->x);CHKERRQ(ierr);
ierr = VecDestroy(&is->y);CHKERRQ(ierr);
ierr = VecScatterDestroy(&is->rctx);CHKERRQ(ierr);
ierr = VecScatterDestroy(&is->cctx);CHKERRQ(ierr);
ierr = MatDestroy(&is->A);CHKERRQ(ierr);
ierr = PetscSFDestroy(&is->sf);CHKERRQ(ierr);
ierr = PetscFree2(is->sf_rootdata,is->sf_leafdata);CHKERRQ(ierr);
/* Setup Layout and set local to global maps */
ierr = PetscLayoutSetUp(A->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->cmap);CHKERRQ(ierr);
ierr = PetscLayoutSetISLocalToGlobalMapping(A->rmap,rmapping);CHKERRQ(ierr);
ierr = PetscLayoutSetISLocalToGlobalMapping(A->cmap,cmapping);CHKERRQ(ierr);
/* Create the local matrix A */
ierr = ISLocalToGlobalMappingGetSize(rmapping,&nr);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetBlockSize(rmapping,&rbs);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(cmapping,&nc);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetBlockSize(cmapping,&cbs);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_SELF,&is->A);CHKERRQ(ierr);
ierr = MatSetType(is->A,MATAIJ);CHKERRQ(ierr);
ierr = MatSetSizes(is->A,nr,nc,nr,nc);CHKERRQ(ierr);
ierr = MatSetBlockSizes(is->A,rbs,cbs);CHKERRQ(ierr);
ierr = MatSetOptionsPrefix(is->A,((PetscObject)A)->prefix);CHKERRQ(ierr);
ierr = MatAppendOptionsPrefix(is->A,"is_");CHKERRQ(ierr);
ierr = MatSetFromOptions(is->A);CHKERRQ(ierr);
/* Create the local work vectors */
ierr = MatCreateVecs(is->A,&is->x,&is->y);CHKERRQ(ierr);
/* setup the global to local scatters */
ierr = MatCreateVecs(A,&cglobal,&rglobal);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,nr,0,1,&to);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApplyIS(rmapping,to,&from);CHKERRQ(ierr);
ierr = VecScatterCreate(rglobal,from,is->y,to,&is->rctx);CHKERRQ(ierr);
if (rmapping != cmapping) {
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,nc,0,1,&to);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApplyIS(cmapping,to,&from);CHKERRQ(ierr);
ierr = VecScatterCreate(cglobal,from,is->x,to,&is->cctx);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)is->rctx);CHKERRQ(ierr);
is->cctx = is->rctx;
}
ierr = VecDestroy(&rglobal);CHKERRQ(ierr);
ierr = VecDestroy(&cglobal);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
