/*@
MatXAIJSetPreallocation - set preallocation for serial and parallel AIJ, BAIJ, and SBAIJ matrices
Collective on Mat
Input Arguments:
+ A - matrix being preallocated
. bs - block size
. dnnz - number of nonzero blocks per block row of diagonal part of parallel matrix
. onnz - number of nonzero blocks per block row of off-diagonal part of parallel matrix
. dnnzu - number of nonzero blocks per block row of upper-triangular part of diagonal part of parallel matrix
- onnzu - number of nonzero blocks per block row of upper-triangular part of off-diagonal part of parallel matrix
Level: beginner
.seealso: MatSeqAIJSetPreallocation(), MatMPIAIJSetPreallocation(), MatSeqBAIJSetPreallocation(), MatMPIBAIJSetPreallocation(), MatSeqSBAIJSetPreallocation(), MatMPISBAIJSetPreallocation(),
PetscSplitOwnership()
@*/
PetscErrorCode MatXAIJSetPreallocation(Mat A,PetscInt bs,const PetscInt dnnz[],const PetscInt onnz[],const PetscInt dnnzu[],const PetscInt onnzu[])
{
PetscErrorCode ierr;
void (*aij)(void);
PetscFunctionBegin;
ierr = MatSetBlockSize(A,bs);
CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->rmap);
CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->cmap);
CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,0,dnnz);
CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,0,dnnz,0,onnz);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,bs,0,dnnzu);
CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,bs,0,dnnzu,0,onnzu);
CHKERRQ(ierr);
/*
In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
good before going on with it.
*/
ierr = PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
CHKERRQ(ierr);
if (!aij) {
ierr = PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
CHKERRQ(ierr);
}
if (aij) {
if (bs == 1) {
ierr = MatSeqAIJSetPreallocation(A,0,dnnz);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz,0,onnz);
CHKERRQ(ierr);
} else { /* Convert block-row precallocation to scalar-row */
PetscInt i,m,*sdnnz,*sonnz;
ierr = MatGetLocalSize(A,&m,NULL);
CHKERRQ(ierr);
ierr = PetscMalloc2((!!dnnz)*m,PetscInt,&sdnnz,(!!onnz)*m,PetscInt,&sonnz);
CHKERRQ(ierr);
for (i=0; i<m; i++) {
if (dnnz) sdnnz[i] = dnnz[i/bs] * bs;
if (onnz) sonnz[i] = onnz[i/bs] * bs;
}
ierr = MatSeqAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL,0,onnz ? sonnz : NULL);
CHKERRQ(ierr);
ierr = PetscFree2(sdnnz,sonnz);
CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
int DA::createActiveMatrix(Mat &M, MatType mtype, unsigned int dof) {
// first determine the size ...
unsigned int sz = 0;
if(m_bIamActive) {
sz = dof*(m_uiNodeSize + m_uiBoundaryNodeSize);
// now create the PETSc Mat
PetscBool isAij, isAijSeq, isAijPrl, isSuperLU, isSuperLU_Dist;
PetscStrcmp(mtype,MATAIJ,&isAij);
PetscStrcmp(mtype,MATSEQAIJ,&isAijSeq);
PetscStrcmp(mtype,MATMPIAIJ,&isAijPrl);
isSuperLU = PETSC_FALSE; //PetscStrcmp(mtype,MATSUPERLU,&isSuperLU);
isSuperLU_Dist = PETSC_FALSE; //PetscStrcmp(mtype,MATSUPERLU_DIST,&isSuperLU_Dist);
MatCreate(m_mpiCommActive, &M);
MatSetSizes(M, sz,sz, PETSC_DECIDE, PETSC_DECIDE);
MatSetType(M,mtype);
if(isAij || isAijSeq || isAijPrl || isSuperLU || isSuperLU_Dist) {
if(m_iNpesActive > 1) {
MatMPIAIJSetPreallocation(M, 53*dof , PETSC_NULL, 53*dof , PETSC_NULL);
}else {
MatSeqAIJSetPreallocation(M, 53*dof , PETSC_NULL);
}
}
}//end if active
return 0;
}//end function
PETSC_EXTERN PetscErrorCode MatGetFactor_seqaij_essl(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
PetscErrorCode ierr;
Mat_Essl *essl;
PetscFunctionBegin;
if (A->cmap->N != A->rmap->N) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"matrix must be square");
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,PETSC_DECIDE,PETSC_DECIDE,A->rmap->n,A->cmap->n);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(B,0,NULL);CHKERRQ(ierr);
ierr = PetscNewLog(B,&essl);CHKERRQ(ierr);
B->spptr = essl;
B->ops->lufactorsymbolic = MatLUFactorSymbolic_Essl;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_essl);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_LU;
ierr = PetscFree(B->solvertype);CHKERRQ(ierr);
ierr = PetscStrallocpy(MATSOLVERESSL,&B->solvertype);CHKERRQ(ierr);
*F = B;
PetscFunctionReturn(0);
}
PetscErrorCode cHamiltonianMatrix::hamiltonianConstruction(){
ierr = MatCreate(PETSC_COMM_WORLD,&Hpolaron);CHKERRQ(ierr);
ierr = MatSetType(Hpolaron,MATMPIAIJ);CHKERRQ(ierr);
ierr = MatSetSizes(Hpolaron,PETSC_DECIDE,PETSC_DECIDE,DIM,DIM);CHKERRQ(ierr);
// TODO: should be able to set the symmetric/hermitian option and
// only do upper-right triangle part of matrix construction .
// and perform corresponding operations thereon.
// ierr = MatSetOption(Hpolaron,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
// ierr = MatSetOption(Hpolaron,MAT_HERMITIAN,PETSC_TRUE);CHKERRQ(ierr);
// TODO: what is the estimate of the pre-allocation?
// -- number of nonzeros per row in DIAGONAL portion of local submatrix
// (same value is used for all local rows) ? I put dim temporarily here.
// number of nonzeros per row in the OFF-DIAGONAL portion of local submatrix
// (same value is used for all local rows) ? I put dim temporarily here..
// More details at http://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/Mat/MatMPIAIJSetPreallocation.html
ierr = MatMPIAIJSetPreallocation(Hpolaron,DIM,NULL,DIM,NULL);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(Hpolaron,DIM,NULL);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Hpolaron,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetLocalSize(Hpolaron,&nlocal, NULL);CHKERRQ(ierr);
ierr = assemblance();CHKERRQ(ierr);
ierr = MatAssemblyBegin(Hpolaron,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Hpolaron,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
// ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_DENSE );CHKERRQ(ierr);
// ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_MATLAB );CHKERRQ(ierr);
// ierr = MatView(Hpolaron, PETSC_VIEWER_STDOUT_WORLD );CHKERRQ(ierr);
return ierr;
}
PETSC_EXTERN PetscErrorCode MatGetFactor_seqaij_lusol(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
Mat_LUSOL *lusol;
PetscErrorCode ierr;
int m, n;
PetscFunctionBegin;
ierr = MatGetSize(A, &m, &n);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,PETSC_DECIDE,PETSC_DECIDE,m,n);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(B,0,NULL);CHKERRQ(ierr);
ierr = PetscNewLog(B,&lusol);CHKERRQ(ierr);
B->spptr = lusol;
B->ops->lufactorsymbolic = MatLUFactorSymbolic_LUSOL;
B->ops->destroy = MatDestroy_LUSOL;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverType_C",MatFactorGetSolverType_seqaij_lusol);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_LU;
ierr = PetscFree(B->solvertype);CHKERRQ(ierr);
ierr = PetscStrallocpy(MATSOLVERLUSOL,&B->solvertype);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Almost a copy of MatGetFactor_seqsbaij_cholmod, yuck */
PETSC_EXTERN PetscErrorCode MatGetFactor_seqaij_cholmod(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
Mat_CHOLMOD *chol;
PetscErrorCode ierr;
PetscInt m=A->rmap->n,n=A->cmap->n;
PetscFunctionBegin;
if (ftype != MAT_FACTOR_CHOLESKY) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"CHOLMOD cannot do %s factorization with AIJ, only %s",
MatFactorTypes[ftype],MatFactorTypes[MAT_FACTOR_CHOLESKY]);
/* Create the factorization matrix F */
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,PETSC_DECIDE,PETSC_DECIDE,m,n);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(B,0,NULL);CHKERRQ(ierr);
ierr = PetscNewLog(B,&chol);CHKERRQ(ierr);
chol->Wrap = MatWrapCholmod_seqaij;
chol->Destroy = MatDestroy_SeqAIJ;
B->spptr = chol;
B->ops->view = MatView_CHOLMOD;
B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_CHOLMOD;
B->ops->destroy = MatDestroy_CHOLMOD;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_seqaij_cholmod);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_CHOLESKY;
B->assembled = PETSC_TRUE; /* required by -ksp_view */
B->preallocated = PETSC_TRUE;
ierr = CholmodStart(B);CHKERRQ(ierr);
*F = B;
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode MatGetFactor_mpiaij_pastix(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
PetscErrorCode ierr;
Mat_Pastix *pastix;
PetscFunctionBegin;
if (ftype != MAT_FACTOR_LU) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot use PETSc AIJ matrices with PaStiX Cholesky, use SBAIJ matrix");
/* Create the factorization matrix */
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(B,0,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(B,0,NULL,0,NULL);CHKERRQ(ierr);
B->ops->lufactorsymbolic = MatLUFactorSymbolic_AIJPASTIX;
B->ops->view = MatView_PaStiX;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_pastix);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_LU;
ierr = PetscNewLog(B,&pastix);CHKERRQ(ierr);
pastix->CleanUpPastix = PETSC_FALSE;
pastix->isAIJ = PETSC_TRUE;
pastix->scat_rhs = NULL;
pastix->scat_sol = NULL;
pastix->Destroy = B->ops->destroy;
B->ops->destroy = MatDestroy_Pastix;
B->spptr = (void*)pastix;
*F = B;
PetscFunctionReturn(0);
}
static PetscErrorCode DMCreateInterpolation_Redundant(DM dmc,DM dmf,Mat *P,Vec *scale)
{
PetscErrorCode ierr;
DM_Redundant *redc = (DM_Redundant*)dmc->data;
DM_Redundant *redf = (DM_Redundant*)dmf->data;
PetscMPIInt flag;
PetscInt i,rstart,rend;
PetscFunctionBegin;
ierr = MPI_Comm_compare(PetscObjectComm((PetscObject)dmc),PetscObjectComm((PetscObject)dmf),&flag);CHKERRQ(ierr);
if (flag != MPI_CONGRUENT && flag != MPI_IDENT) SETERRQ(PetscObjectComm((PetscObject)dmf),PETSC_ERR_SUP,"cannot change communicators");
if (redc->rank != redf->rank) SETERRQ(PetscObjectComm((PetscObject)dmf),PETSC_ERR_ARG_INCOMP,"Owning rank does not match");
if (redc->N != redf->N) SETERRQ(PetscObjectComm((PetscObject)dmf),PETSC_ERR_ARG_INCOMP,"Global size does not match");
ierr = MatCreate(PetscObjectComm((PetscObject)dmc),P);CHKERRQ(ierr);
ierr = MatSetSizes(*P,redc->n,redc->n,redc->N,redc->N);CHKERRQ(ierr);
ierr = MatSetType(*P,MATAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*P,1,0);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*P,1,0,0,0);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(*P,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {ierr = MatSetValue(*P,i,i,1.0,INSERT_VALUES);CHKERRQ(ierr);}
ierr = MatAssemblyBegin(*P,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (scale) {ierr = DMCreateInterpolationScale(dmc,dmf,*P,scale);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
/***************** Array Access ********************/
int DA::createMatrix(Mat &M, MatType mtype, unsigned int dof) {
// first determine the size ...
unsigned int sz = 0;
if(m_bIamActive) {
sz = dof*(m_uiNodeSize + m_uiBoundaryNodeSize);
}//end if active
// now create the PETSc Mat
// The "parallel direct solver" matrix types like MATAIJSPOOLES are ALL gone in petsc-3.0.0
// Thus, I (Ilya Lashuk) "delete" all such checks for matrix type. Hope it is reasonable thing to do.
PetscBool isAij, isAijSeq, isAijPrl, isSuperLU, isSuperLU_Dist;
PetscStrcmp(mtype,MATAIJ,&isAij);
PetscStrcmp(mtype,MATSEQAIJ,&isAijSeq);
PetscStrcmp(mtype,MATMPIAIJ,&isAijPrl);
isSuperLU = PETSC_FALSE; // PetscStrcmp(mtype,MATSUPERLU,&isSuperLU);
isSuperLU_Dist = PETSC_FALSE; // PetscStrcmp(mtype,MATSUPERLU_DIST,&isSuperLU_Dist);
MatCreate(m_mpiCommAll, &M);
MatSetSizes(M, sz,sz, PETSC_DECIDE, PETSC_DECIDE);
MatSetType(M,mtype);
if(isAij || isAijSeq || isAijPrl || isSuperLU || isSuperLU_Dist) {
if(m_iNpesAll > 1) {
MatMPIAIJSetPreallocation(M, 53*dof , PETSC_NULL, 53*dof , PETSC_NULL);
}else {
MatSeqAIJSetPreallocation(M, 53*dof , PETSC_NULL);
}
}
return 0;
}//end function
int main(int argc, char **args)
{
Mat A;
MatPartitioning part;
IS is;
PetscInt r,N = 10, start, end;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL, "-N", &N, PETSC_NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD, &A);CHKERRQ(ierr);
ierr = MatSetSizes(A, PETSC_DECIDE, PETSC_DECIDE, N, N);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A, 3, PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A, 3, PETSC_NULL, 2, PETSC_NULL);CHKERRQ(ierr);
// Create a linear mesh
ierr = MatGetOwnershipRange(A, &start, &end);CHKERRQ(ierr);
for(r = start; r < end; ++r) {
if (r == 0) {
PetscInt cols[2];
PetscScalar vals[2];
cols[0] = r; cols[1] = r+1;
vals[0] = 1.0; vals[1] = 1.0;
ierr = MatSetValues(A, 1, &r, 2, cols, vals, INSERT_VALUES);CHKERRQ(ierr);
} else if (r == N-1) {
PetscInt cols[2];
PetscScalar vals[2];
cols[0] = r-1; cols[1] = r;
vals[0] = 1.0; vals[1] = 1.0;
ierr = MatSetValues(A, 1, &r, 2, cols, vals, INSERT_VALUES);CHKERRQ(ierr);
} else {
PetscInt cols[3];
PetscScalar vals[3];
cols[0] = r-1; cols[1] = r; cols[2] = r+1;
vals[0] = 1.0; vals[1] = 1.0; vals[2] = 1.0;
ierr = MatSetValues(A, 1, &r, 3, cols, vals, INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatPartitioningCreate(PETSC_COMM_WORLD, &part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part, A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
//ierr = MatPartitioningSetVertexWeights(part, const PetscInt weights[]);CHKERRQ(ierr);
//ierr = MatPartitioningSetPartitionWeights(part,const PetscReal weights[]);CHKERRQ(ierr);
ierr = MatPartitioningApply(part, &is);CHKERRQ(ierr);
ierr = ISView(is, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc, char **args)
{
Mat A;
MatPartitioning part;
IS is;
PetscInt i,m,N,rstart,rend,nemptyranks,*emptyranks,nbigranks,*bigranks;
PetscMPIInt rank,size;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
nemptyranks = 10;
nbigranks = 10;
ierr = PetscMalloc2(nemptyranks,PetscInt,&emptyranks,nbigranks,PetscInt,&bigranks);CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,PETSC_NULL,"Partitioning example options",PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsIntArray("-emptyranks","Ranks to be skipped by partition","",emptyranks,&nemptyranks,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsIntArray("-bigranks","Ranks to be overloaded","",bigranks,&nbigranks,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
m = 1;
for (i=0; i<nemptyranks; i++) if (rank == emptyranks[i]) m = 0;
for (i=0; i<nbigranks; i++) if (rank == bigranks[i]) m = 5;
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m,m,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,1,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,1,3,PETSC_NULL,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,1,2,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,1,2,PETSC_NULL,1,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetSize(A,PETSC_NULL,&N);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
const PetscInt cols[] = {(i+N-1)%N,i,(i+1)%N};
const PetscScalar vals[] = {1,1,1};
ierr = MatSetValues(A,1,&i,3,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFree2(emptyranks,bigranks);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode ConvertMatrixToMat(MPI_Comm comm,matrix *B,Mat *PB)
{
PetscMPIInt size,rank;
PetscErrorCode ierr;
int m,n,M,N;
int d_nz,o_nz;
int *d_nnz,*o_nnz;
int i,k,global_row,global_col,first_diag_col,last_diag_col;
PetscScalar val;
PetscFunctionBegin;
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
m = n = B->mnls[rank];
d_nz = o_nz = 0;
/* Determine preallocation for MatCreateMPIAIJ */
ierr = PetscMalloc(m*sizeof(PetscInt),&d_nnz);CHKERRQ(ierr);
ierr = PetscMalloc(m*sizeof(PetscInt),&o_nnz);CHKERRQ(ierr);
for (i=0; i<m; i++) d_nnz[i] = o_nnz[i] = 0;
first_diag_col = B->start_indices[rank];
last_diag_col = first_diag_col + B->mnls[rank];
for (i=0; i<B->mnls[rank]; i++) {
for (k=0; k<B->lines->len[i]; k++) {
global_col = B->lines->ptrs[i][k];
if ((global_col >= first_diag_col) && (global_col < last_diag_col))
d_nnz[i]++;
else
o_nnz[i]++;
}
}
M = N = B->n;
/* Here we only know how to create AIJ format */
ierr = MatCreate(comm,PB);CHKERRQ(ierr);
ierr = MatSetSizes(*PB,m,n,M,N);CHKERRQ(ierr);
ierr = MatSetType(*PB,MATAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*PB,d_nz,d_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*PB,d_nz,d_nnz,o_nz,o_nnz);CHKERRQ(ierr);
for (i=0; i<B->mnls[rank]; i++) {
global_row = B->start_indices[rank]+i;
for (k=0; k<B->lines->len[i]; k++) {
global_col = B->lines->ptrs[i][k];
val = B->lines->A[i][k];
ierr = MatSetValues(*PB,1,&global_row,1,&global_col,&val,ADD_VALUES);CHKERRQ(ierr);
}
}
ierr = PetscFree(d_nnz);CHKERRQ(ierr);
ierr = PetscFree(o_nnz);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*PB,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*PB,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "MatConvert_SeqBAIJ_SeqAIJ"
PetscErrorCode MatConvert_SeqBAIJ_SeqAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat B;
Mat_SeqBAIJ *a = (Mat_SeqBAIJ*)A->data;
PetscErrorCode ierr;
PetscInt bs = A->rmap->bs,*ai = a->i,*aj = a->j,n = A->rmap->N/bs,i,j,k;
PetscInt *rowlengths,*rows,*cols,maxlen = 0,ncols;
MatScalar *aa = a->a;
PetscFunctionBegin;
ierr = PetscMalloc(n*bs*sizeof(PetscInt),&rowlengths);CHKERRQ(ierr);
for (i=0; i<n; i++) {
maxlen = PetscMax(maxlen,(ai[i+1] - ai[i]));
for (j=0; j<bs; j++) {
rowlengths[i*bs+j] = bs*(ai[i+1] - ai[i]);
}
}
ierr = MatCreate(((PetscObject)A)->comm,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);CHKERRQ(ierr);
ierr = MatSetType(B,MATSEQAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(B,0,rowlengths);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_ROW_ORIENTED,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscFree(rowlengths);CHKERRQ(ierr);
ierr = PetscMalloc(bs*sizeof(PetscInt),&rows);CHKERRQ(ierr);
ierr = PetscMalloc(bs*maxlen*sizeof(PetscInt),&cols);CHKERRQ(ierr);
for (i=0; i<n; i++) {
for (j=0; j<bs; j++) {
rows[j] = i*bs+j;
}
ncols = ai[i+1] - ai[i];
for (k=0; k<ncols; k++) {
for (j=0; j<bs; j++) {
cols[k*bs+j] = bs*(*aj) + j;
}
aj++;
}
ierr = MatSetValues(B,bs,rows,bs*ncols,cols,aa,INSERT_VALUES);CHKERRQ(ierr);
aa += ncols*bs*bs;
}
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
B->rmap->bs = A->rmap->bs;
if (reuse == MAT_REUSE_MATRIX) {
ierr = MatHeaderReplace(A,B);CHKERRQ(ierr);
} else {
*newmat = B;
}
PetscFunctionReturn(0);
}
/*
Takes the local part of an already assembled MPIAIJ matrix
and disassembles it. This is to allow new nonzeros into the matrix
that require more communication in the matrix vector multiply.
Thus certain data-structures must be rebuilt.
Kind of slow! But that's what application programmers get when
they are sloppy.
*/
PetscErrorCode MatDisAssemble_MPIAIJ(Mat A)
{
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)A->data;
Mat B = aij->B,Bnew;
Mat_SeqAIJ *Baij = (Mat_SeqAIJ*)B->data;
PetscErrorCode ierr;
PetscInt i,j,m = B->rmap->n,n = A->cmap->N,col,ct = 0,*garray = aij->garray,*nz,ec;
PetscScalar v;
PetscFunctionBegin;
/* free stuff related to matrix-vec multiply */
ierr = VecGetSize(aij->lvec,&ec);CHKERRQ(ierr); /* needed for PetscLogObjectMemory below */
ierr = VecDestroy(&aij->lvec);CHKERRQ(ierr);
ierr = VecScatterDestroy(&aij->Mvctx);CHKERRQ(ierr);
if (aij->colmap) {
#if defined (PETSC_USE_CTABLE)
ierr = PetscTableDestroy(&aij->colmap);CHKERRQ(ierr);
#else
ierr = PetscFree(aij->colmap);CHKERRQ(ierr);
ierr = PetscLogObjectMemory(A,-aij->B->cmap->n*sizeof(PetscInt));CHKERRQ(ierr);
#endif
}
/* make sure that B is assembled so we can access its values */
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* invent new B and copy stuff over */
ierr = PetscMalloc((m+1)*sizeof(PetscInt),&nz);CHKERRQ(ierr);
for (i=0; i<m; i++) {
nz[i] = Baij->i[i+1] - Baij->i[i];
}
ierr = MatCreate(PETSC_COMM_SELF,&Bnew);CHKERRQ(ierr);
ierr = MatSetSizes(Bnew,m,n,m,n);CHKERRQ(ierr);
ierr = MatSetBlockSizes(Bnew,A->rmap->bs,A->cmap->bs);CHKERRQ(ierr);
ierr = MatSetType(Bnew,((PetscObject)B)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(Bnew,0,nz);CHKERRQ(ierr);
((Mat_SeqAIJ*)Bnew->data)->nonew = Baij->nonew; /* Inherit insertion error options. */
ierr = PetscFree(nz);CHKERRQ(ierr);
for (i=0; i<m; i++) {
for (j=Baij->i[i]; j<Baij->i[i+1]; j++) {
col = garray[Baij->j[ct]];
v = Baij->a[ct++];
ierr = MatSetValues(Bnew,1,&i,1,&col,&v,B->insertmode);CHKERRQ(ierr);
}
}
ierr = PetscFree(aij->garray);CHKERRQ(ierr);
ierr = PetscLogObjectMemory(A,-ec*sizeof(PetscInt));CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscLogObjectParent(A,Bnew);CHKERRQ(ierr);
aij->B = Bnew;
A->was_assembled = PETSC_FALSE;
PetscFunctionReturn(0);
}
PetscErrorCode AssemblePressureMatrx( UserContext* uc )
{
PetscErrorCode ierr;
PetscScalar val[5];
PetscInt i, j;
BCNode *bcn;
Node *n;
PetscFunctionBegin;
ierr = MatCreate(PETSC_COMM_WORLD, &uc->A); CHKERRQ(ierr);
ierr = MatSetSizes(uc->A, PETSC_DECIDE, PETSC_DECIDE, uc->numNodes, uc->numNodes); CHKERRQ(ierr);
ierr = MatSetType(uc->A, MATUMFPACK); CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(uc->A, 5, PETSC_NULL); CHKERRQ(ierr);
// ierr = MatCreateSeqSBAIJ(PETSC_COMM_WORLD,1,uc->numNodes, uc->numNodes,3, PETSC_NULL, &uc->A); CHKERRQ(ierr);
// ierr = MatSetOption(uc->A, MAT_SYMMETRIC); CHKERRQ(ierr);
// ierr = MatSetOption(uc->A, MAT_USE_INODES); CHKERRQ(ierr);
// ierr = MatSetOption(uc->A, MAT_IGNORE_LOWER_TRIANGULAR); CHKERRQ(ierr);
for (i = 0; i < uc->numNodes; ++i)
{
n = &uc->nodes[i];
for (j = 0; j < n->numNei; ++j)
{
val[j] = negone;
}
val[n->numNei] = n->numNei;
ierr = MatSetValues(uc->A, 1, &i, n->numNei+1, n->nei, val, INSERT_VALUES); CHKERRQ(ierr);
}
PetscInt idx;
for( i = 0; i < uc->numBC; ++i)
{
idx = uc->imageToNode[uc->bcToImage[i]];
n = &uc->nodes[idx];
for( j = 0; j < n->numNei; ++j)
{
ierr = MatSetValue(uc->A, idx, n->nei[j], zero, INSERT_VALUES); CHKERRQ(ierr);
ierr = MatSetValue(uc->A, n->nei[j], idx, zero, INSERT_VALUES); CHKERRQ(ierr);
}
ierr = MatSetValue(uc->A, idx, idx, one, INSERT_VALUES); CHKERRQ(ierr);
}
ierr = MatSetOption(uc->A, MAT_NEW_NONZERO_LOCATION_ERR); CHKERRQ(ierr);
ierr = MatSetOption(uc->A, MAT_NO_NEW_NONZERO_LOCATIONS); CHKERRQ(ierr);
ierr = MatAssemblyBegin(uc->A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(uc->A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatStoreValues(uc->A); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode MatGetFactor_seqaij_klu(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
Mat_KLU *lu;
PetscErrorCode ierr;
PetscInt m=A->rmap->n,n=A->cmap->n,idx,status;
PetscBool flg;
PetscFunctionBegin;
/* Create the factorization matrix F */
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,PETSC_DECIDE,PETSC_DECIDE,m,n);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(B,0,NULL);CHKERRQ(ierr);
ierr = PetscNewLog(B,&lu);CHKERRQ(ierr);
B->spptr = lu;
B->ops->lufactorsymbolic = MatLUFactorSymbolic_KLU;
B->ops->destroy = MatDestroy_KLU;
B->ops->view = MatView_KLU;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_seqaij_klu);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_LU;
B->assembled = PETSC_TRUE; /* required by -ksp_view */
B->preallocated = PETSC_TRUE;
/* initializations */
/* ------------------------------------------------*/
/* get the default control parameters */
status = klu_K_defaults(&lu->Common);
if(status <= 0) {
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"KLU Initialization failed");
}
lu->Common.scale = 0; /* No row scaling */
ierr = PetscOptionsBegin(PetscObjectComm((PetscObject)A),((PetscObject)A)->prefix,"KLU Options","Mat");CHKERRQ(ierr);
/* Partial pivoting tolerance */
ierr = PetscOptionsReal("-mat_klu_pivot_tol","Partial pivoting tolerance","None",lu->Common.tol,&lu->Common.tol,NULL);CHKERRQ(ierr);
/* BTF pre-ordering */
ierr = PetscOptionsInt("-mat_klu_use_btf","Enable BTF preordering","None",lu->Common.btf,&lu->Common.btf,NULL);CHKERRQ(ierr);
/* Matrix reordering */
ierr = PetscOptionsEList("-mat_klu_ordering","Internal ordering method","None",KluOrderingTypes,sizeof(KluOrderingTypes)/sizeof(KluOrderingTypes[0]),KluOrderingTypes[0],&idx,&flg);CHKERRQ(ierr);
if (flg) {
if ((int)idx == 2) lu->PetscMatOrdering = PETSC_TRUE; /* use Petsc mat ordering (note: size is for the transpose, and PETSc r = Klu perm_c) */
else lu->Common.ordering = (int)idx;
}
/* Matrix row scaling */
ierr = PetscOptionsEList("-mat_klu_row_scale","Matrix row scaling","None",scale,3,scale[0],&idx,&flg);CHKERRQ(ierr);
PetscOptionsEnd();
*F = B;
PetscFunctionReturn(0);
}
void SingleLongPipe::initialize() {
MatCreate(PETSC_COMM_WORLD, &M);
MatSetSizes(M, local_n, local_n, global_n, global_n);
MatSetFromOptions(M);
MatMPIAIJSetPreallocation(M, 2, nullptr, 2, nullptr);
MatSeqAIJSetPreallocation(M, 2, nullptr);
MatSetUp(M);
KSPCreate(PETSC_COMM_WORLD, &ksp);
KSPSetOperators(ksp, M, M);
KSPSetFromOptions(ksp);
KSPSetUp(ksp);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscSF sf;
PetscInt bs,i,nroots,*rootdata,nleaves,*leafdata,nlocalcols;
const PetscInt *gidxs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) {
SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
}
ierr = MatGetLocalSize(B,&nroots,NULL);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&nleaves,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscCalloc2(nroots,&rootdata,nleaves,&leafdata);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)B),&sf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,B->rmap,nleaves,NULL,PETSC_COPY_VALUES,gidxs);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
if (!d_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += d_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += o_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves;i++) {
leafdata[i] = PetscMin(leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree2(rootdata,leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int SFieldBeginRuns(SField sfv,
unsigned int N,
const unsigned int *nelem) {
mySField sf = static_cast<mySField>(sfv);
unsigned int j;
assert(!sf->running);
sf->maxN = N;
sf->running = 1;
sf->timeAssembly = 0;
sf->timeSolver = 0;
// Given k indices we need k*2^d variables
// Given N variables we need N / 2^d
// Now we are overestimating the number of modes.
sf->modes = sf->maxN;//(sf->N+ignored_modes) / (1 << sf->d) + ((sf->N+ignored_modes) % (1 << sf->d) != 0);
sf->N_multi_idx = new ind_t[sf->modes * sf->d];
GenTDSet(sf->d, 0, sf->N_multi_idx, sf->modes);
// Create sparse Matrix of size prod(mesh)
Mat J; Vec F; Vec U;
int s=1;
for (j=0;j < sf->d;j++){
sf->mesh[j] = nelem[j];
s *= sf->mesh[j];
}
MatCreate(PETSC_COMM_WORLD,&J);
MatSetSizes(J,s,s,s,s);
MatSetType(J,MATSEQAIJ);
MatSeqAIJSetPreallocation(J,1+2*sf->d,NULL);
MatSetFromOptions(J);
/* MatSetType(J,MATSEQDENSE); */
/* MatSeqDenseSetPreallocation(J,NULL); */
/* MatSetFromOptions(J); */
MatSetUp(J);
VecCreate(PETSC_COMM_WORLD,&F);
VecSetSizes(F,PETSC_DECIDE,s);
VecSetFromOptions(F);
VecSetUp(F);
VecDuplicate(F,&U);
KSP ksp;
KSPCreate(PETSC_COMM_WORLD,&ksp);
KSPSetFromOptions(ksp);
sf->J = J; sf->F = F; sf->U = U; sf->ksp = ksp;
return 0;
}
PETSC_EXTERN PetscErrorCode MatGetFactor_aij_mkl_pardiso(Mat A,MatFactorType ftype,Mat *F) {
Mat B;
PetscErrorCode ierr;
Mat_MKL_PARDISO *mat_mkl_pardiso;
PetscBool isSeqAIJ;
PetscFunctionBegin;
/* Create the factorization matrix */
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isSeqAIJ);
CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);
CHKERRQ(ierr);
ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);
CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);
CHKERRQ(ierr);
if (isSeqAIJ) {
ierr = MatSeqAIJSetPreallocation(B,0,NULL);
CHKERRQ(ierr);
} else {
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Is not allowed other types of matrices apart from MATSEQAIJ.");
}
B->ops->lufactorsymbolic = MatLUFactorSymbolic_AIJMKL_PARDISO;
B->ops->destroy = MatDestroy_MKL_PARDISO;
B->ops->view = MatView_MKL_PARDISO;
B->factortype = ftype;
B->ops->getinfo = MatGetInfo_MKL_PARDISO;
B->assembled = PETSC_TRUE; /* required by -ksp_view */
ierr = PetscNewLog(B,&mat_mkl_pardiso);
CHKERRQ(ierr);
B->spptr = mat_mkl_pardiso;
ierr = PetscObjectComposeFunction((PetscObject)B,"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_mkl_pardiso);
CHKERRQ(ierr);
ierr = PetscObjectComposeFunction((PetscObject)B,"MatMkl_PardisoSetCntl_C",MatMkl_PardisoSetCntl_MKL_PARDISO);
CHKERRQ(ierr);
ierr = PetscInitializeMKL_PARDISO(A, mat_mkl_pardiso);
CHKERRQ(ierr);
*F = B;
PetscFunctionReturn(0);
}
PetscErrorCode PreallocateJacobian(Mat J, Userctx *user)
{
PetscErrorCode ierr;
PetscInt *d_nnz;
PetscInt i,idx=0,start=0;
PetscFunctionBegin;
ierr = PetscMalloc1(user->neqs_pgrid,&d_nnz);CHKERRQ(ierr);
for (i=0; i<user->neqs_pgrid; i++) d_nnz[i] = 0;
/* Generator subsystem */
for (i=0; i < ngen; i++) {
d_nnz[idx] += 3;
d_nnz[idx+1] += 2;
d_nnz[idx+2] += 2;
d_nnz[idx+3] += 5;
d_nnz[idx+4] += 6;
d_nnz[idx+5] += 6;
d_nnz[user->neqs_gen+2*gbus[i]] += 3;
d_nnz[user->neqs_gen+2*gbus[i]+1] += 3;
d_nnz[idx+6] += 2;
d_nnz[idx+7] += 2;
d_nnz[idx+8] += 5;
idx = idx + 9;
}
start = user->neqs_gen;
PetscInt ncols;
for (i=0; i < nbus; i++) {
ierr = MatGetRow(user->Ybus,2*i,&ncols,NULL,NULL);CHKERRQ(ierr);
d_nnz[start+2*i] += ncols;
d_nnz[start+2*i+1] += ncols;
ierr = MatRestoreRow(user->Ybus,2*i,&ncols,NULL,NULL);CHKERRQ(ierr);
}
ierr = MatSeqAIJSetPreallocation(J,0,d_nnz);CHKERRQ(ierr);
ierr = PetscFree(d_nnz);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void linearSystemPETSc<scalar>::preAllocateEntries()
{
if (_entriesPreAllocated) return;
if (!_isAllocated) Msg::Fatal("system must be allocated first");
int blockSize = _getBlockSizeFromParameters();
std::vector<int> nByRowDiag (_localSize), nByRowOffDiag (_localSize);
if (_sparsity.getNbRows() == 0) {
PetscInt prealloc = 500;
PetscBool set;
PetscOptionsGetInt(PETSC_NULL, "-petsc_prealloc", &prealloc, &set);
prealloc = std::min(prealloc, _localSize);
nByRowDiag.resize(0);
nByRowDiag.resize(_localSize, prealloc);
} else {
for (int i = 0; i < _localSize; i++) {
int n;
const int *r = _sparsity.getRow(i, n);
for (int j = 0; j < n; j++) {
if (r[j] >= _localRowStart && r[j] < _localRowEnd)
nByRowDiag[i] ++;
else
nByRowOffDiag[i] ++;
}
}
_sparsity.clear();
}
//MatXAIJSetPreallocation is not available in petsc < 3.3
int commSize = 1;
MPI_Comm_size(_comm, &commSize);
if (commSize == 1){
if (blockSize == 1)
_try(MatSeqAIJSetPreallocation(_a, 0, &nByRowDiag[0]));
else
_try(MatSeqBAIJSetPreallocation(_a, blockSize, 0, &nByRowDiag[0]));
}
else {
if (blockSize == 1)
_try(MatMPIAIJSetPreallocation(_a, 0, &nByRowDiag[0], 0, &nByRowOffDiag[0]));
else
_try(MatMPIBAIJSetPreallocation(_a, blockSize, 0, &nByRowDiag[0], 0, &nByRowOffDiag[0]));
}
if (blockSize > 1)
_try(MatSetOption(_a, MAT_ROW_ORIENTED, PETSC_FALSE));
_entriesPreAllocated = true;
}
void PETScLinearSolver::MatrixCreate( PetscInt m, PetscInt n)
{
MatCreate(PETSC_COMM_WORLD, &A);
// TEST MatSetSizes(A, m_size_loc, PETSC_DECIDE, m, n);
MatSetSizes(A, PETSC_DECIDE, PETSC_DECIDE, m, n);
//MatSetSizes(A, m_size_loc, PETSC_DECIDE, m, n);
MatSetType(A, MATMPIAIJ);
MatSetFromOptions(A);
MatSeqAIJSetPreallocation(A, d_nz, PETSC_NULL);
MatMPIAIJSetPreallocation(A, d_nz, PETSC_NULL, o_nz, PETSC_NULL);
MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);
MatSetUp(A); // KG44 this seems to work with petsc 3.3 ..the commands below result in problems when assembling the matrix with version 3.3
MatGetOwnershipRange(A, &i_start, &i_end);
}
PETSC_EXTERN PetscErrorCode MatGetFactor_seqaij_matlab(Mat A,MatFactorType ftype,Mat *F)
{
PetscErrorCode ierr;
PetscFunctionBegin;
if (A->cmap->N != A->rmap->N) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"matrix must be square");
ierr = MatCreate(PetscObjectComm((PetscObject)A),F);CHKERRQ(ierr);
ierr = MatSetSizes(*F,A->rmap->n,A->cmap->n,A->rmap->n,A->cmap->n);CHKERRQ(ierr);
ierr = MatSetType(*F,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*F,0,NULL);CHKERRQ(ierr);
(*F)->ops->lufactorsymbolic = MatLUFactorSymbolic_Matlab;
(*F)->ops->ilufactorsymbolic = MatLUFactorSymbolic_Matlab;
ierr = PetscObjectComposeFunction((PetscObject)(*F),"MatFactorGetSolverPackage_C",MatFactorGetSolverPackage_seqaij_matlab);CHKERRQ(ierr);
(*F)->factortype = ftype;
PetscFunctionReturn(0);
}
/*@C
MatCreateMPIAIJPERM - Creates a sparse parallel matrix whose local
portions are stored as SEQAIJPERM matrices (a matrix class that inherits
from SEQAIJ but includes some optimizations to allow more effective
vectorization). The same guidelines that apply to MPIAIJ matrices for
preallocating the matrix storage apply here as well.
Collective on MPI_Comm
Input Parameters:
+ comm - MPI communicator
. m - number of local rows (or PETSC_DECIDE to have calculated if M is given)
This value should be the same as the local size used in creating the
y vector for the matrix-vector product y = Ax.
. n - This value should be the same as the local size used in creating the
x vector for the matrix-vector product y = Ax. (or PETSC_DECIDE to have
calculated if N is given) For square matrices n is almost always m.
. M - number of global rows (or PETSC_DETERMINE to have calculated if m is given)
. N - number of global columns (or PETSC_DETERMINE to have calculated if n is given)
. d_nz - number of nonzeros per row in DIAGONAL portion of local submatrix
(same value is used for all local rows)
. d_nnz - array containing the number of nonzeros in the various rows of the
DIAGONAL portion of the local submatrix (possibly different for each row)
or PETSC_NULL, if d_nz is used to specify the nonzero structure.
The size of this array is equal to the number of local rows, i.e 'm'.
For matrices you plan to factor you must leave room for the diagonal entry and
put in the entry even if it is zero.
. o_nz - number of nonzeros per row in the OFF-DIAGONAL portion of local
submatrix (same value is used for all local rows).
- o_nnz - array containing the number of nonzeros in the various rows of the
OFF-DIAGONAL portion of the local submatrix (possibly different for
each row) or PETSC_NULL, if o_nz is used to specify the nonzero
structure. The size of this array is equal to the number
of local rows, i.e 'm'.
Output Parameter:
. A - the matrix
Notes:
If the *_nnz parameter is given then the *_nz parameter is ignored
m,n,M,N parameters specify the size of the matrix, and its partitioning across
processors, while d_nz,d_nnz,o_nz,o_nnz parameters specify the approximate
storage requirements for this matrix.
If PETSC_DECIDE or PETSC_DETERMINE is used for a particular argument on one
processor than it must be used on all processors that share the object for
that argument.
The user MUST specify either the local or global matrix dimensions
(possibly both).
The parallel matrix is partitioned such that the first m0 rows belong to
process 0, the next m1 rows belong to process 1, the next m2 rows belong
to process 2 etc.. where m0,m1,m2... are the input parameter 'm'.
The DIAGONAL portion of the local submatrix of a processor can be defined
as the submatrix which is obtained by extraction the part corresponding
to the rows r1-r2 and columns r1-r2 of the global matrix, where r1 is the
first row that belongs to the processor, and r2 is the last row belonging
to the this processor. This is a square mxm matrix. The remaining portion
of the local submatrix (mxN) constitute the OFF-DIAGONAL portion.
If o_nnz, d_nnz are specified, then o_nz, and d_nz are ignored.
When calling this routine with a single process communicator, a matrix of
type SEQAIJPERM is returned. If a matrix of type MPIAIJPERM is desired
for this type of communicator, use the construction mechanism:
MatCreate(...,&A); MatSetType(A,MPIAIJ); MatMPIAIJSetPreallocation(A,...);
By default, this format uses inodes (identical nodes) when possible.
We search for consecutive rows with the same nonzero structure, thereby
reusing matrix information to achieve increased efficiency.
Options Database Keys:
+ -mat_no_inode - Do not use inodes
. -mat_inode_limit <limit> - Sets inode limit (max limit=5)
- -mat_aij_oneindex - Internally use indexing starting at 1
rather than 0. Note that when calling MatSetValues(),
the user still MUST index entries starting at 0!
Level: intermediate
.keywords: matrix, cray, sparse, parallel
.seealso: MatCreate(), MatCreateSeqAIJPERM(), MatSetValues()
@*/
PetscErrorCode MatCreateMPIAIJPERM(MPI_Comm comm,PetscInt m,PetscInt n,PetscInt M,PetscInt N,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[],Mat *A)
{
PetscErrorCode ierr;
PetscMPIInt size;
PetscFunctionBegin;
ierr = MatCreate(comm,A);CHKERRQ(ierr);
ierr = MatSetSizes(*A,m,n,M,N);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size > 1) {
ierr = MatSetType(*A,MATMPIAIJPERM);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*A,d_nz,d_nnz,o_nz,o_nnz);CHKERRQ(ierr);
} else {
ierr = MatSetType(*A,MATSEQAIJPERM);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*A,d_nz,d_nnz);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
EXTERN_C_END
EXTERN_C_BEGIN
/*
The seq and mpi versions of this function are the same
*/
#undef __FUNCT__
#define __FUNCT__ "MatGetFactor_seqaij_pastix"
PetscErrorCode MatGetFactor_seqaij_pastix(Mat A,MatFactorType ftype,Mat *F)
{
Mat B;
PetscErrorCode ierr;
Mat_Pastix *pastix;
PetscFunctionBegin;
if (ftype != MAT_FACTOR_LU) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot use PETSc AIJ matrices with PaStiX Cholesky, use SBAIJ matrix");
/* Create the factorization matrix */
ierr = MatCreate(((PetscObject)A)->comm,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,A->rmap->n,A->cmap->n,A->rmap->N,A->cmap->N);CHKERRQ(ierr);
ierr = MatSetType(B,((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(B,0,PETSC_NULL);CHKERRQ(ierr);
B->ops->lufactorsymbolic = MatLUFactorSymbolic_AIJPASTIX;
B->ops->view = MatView_PaStiX;
B->ops->getinfo = MatGetInfo_PaStiX;
ierr = PetscObjectComposeFunctionDynamic((PetscObject)B,"MatFactorGetSolverPackage_C","MatFactorGetSolverPackage_pastix", MatFactorGetSolverPackage_pastix);CHKERRQ(ierr);
B->factortype = MAT_FACTOR_LU;
ierr = PetscNewLog(B,Mat_Pastix,&pastix);CHKERRQ(ierr);
pastix->CleanUpPastix = PETSC_FALSE;
pastix->isAIJ = PETSC_TRUE;
pastix->scat_rhs = PETSC_NULL;
pastix->scat_sol = PETSC_NULL;
pastix->Destroy = B->ops->destroy;
B->ops->destroy = MatDestroy_Pastix;
B->spptr = (void*)pastix;
*F = B;
PetscFunctionReturn(0);
}
static PetscErrorCode preallocation(Mat M,PetscInt *d_nz, PetscInt *o_nz) {
PetscErrorCode ierr;
PetscBool isaij,ismpiaij,isseqaij;
PetscMPIInt size;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)M,MATAIJ,&isaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATMPIAIJ,&ismpiaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATSEQAIJ,&isseqaij);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if ((isaij && size == 1) || isseqaij) {
ierr = MatSeqAIJSetPreallocation(M,0,d_nz);CHKERRQ(ierr);
} else if (isaij || ismpiaij) {
ierr = MatMPIAIJSetPreallocation(M,0,d_nz,0,o_nz);CHKERRQ(ierr);
} else {
ierr = PetscInfo(M,"NOT using preallocation\n");CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscInt bs,i,nlocalcols;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
ierr = MatISComputeSF_Private(B);CHKERRQ(ierr);
}
if (!d_nnz) {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nz;
} else {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nz;
} else {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,NULL,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves;i++) {
matis->sf_leafdata[i] = PetscMin(matis->sf_leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves/bs;i++) {
matis->sf_leafdata[i] = matis->sf_leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves/bs;i++) {
matis->sf_leafdata[i] = matis->sf_leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
//STARTPREALLOC
PetscErrorCode Preallocation(Mat J, unfemCtx *user) {
PetscErrorCode ierr;
const int *abfn, *ae, *en;
int *nnz, n, k, l;
nnz = (int *)malloc(sizeof(int)*(user->mesh->N));
ierr = ISGetIndices(user->mesh->bfn,&abfn); CHKERRQ(ierr);
for (n = 0; n < user->mesh->N; n++)
nnz[n] = (abfn[n] == 1) ? 2 : 1;
ierr = ISGetIndices(user->mesh->e,&ae); CHKERRQ(ierr);
for (k = 0; k < user->mesh->K; k++) {
en = ae + 3*k; // en[0], en[1], en[2] are nodes of element k
for (l = 0; l < 3; l++)
if (abfn[en[l]] != 2)
nnz[en[l]] += 1;
}
ierr = ISRestoreIndices(user->mesh->e,&ae); CHKERRQ(ierr);
ierr = ISRestoreIndices(user->mesh->bfn,&abfn); CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(J,-1,nnz); CHKERRQ(ierr);
free(nnz);
return 0;
}
static PetscErrorCode DMCreateMatrix_Redundant(DM dm,Mat *J)
{
DM_Redundant *red = (DM_Redundant*)dm->data;
PetscErrorCode ierr;
ISLocalToGlobalMapping ltog,ltogb;
PetscInt i,rstart,rend,*cols;
PetscScalar *vals;
PetscFunctionBegin;
ierr = MatCreate(PetscObjectComm((PetscObject)dm),J);CHKERRQ(ierr);
ierr = MatSetSizes(*J,red->n,red->n,red->N,red->N);CHKERRQ(ierr);
ierr = MatSetType(*J,dm->mattype);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*J,red->n,NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(*J,1,red->n,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,red->n,NULL,red->N-red->n,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(*J,1,red->n,NULL,red->N-red->n,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(dm,<og);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMappingBlock(dm,<ogb);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMappingBlock(*J,ltogb,ltogb);CHKERRQ(ierr);
ierr = PetscMalloc2(red->N,&cols,red->N,&vals);CHKERRQ(ierr);
for (i=0; i<red->N; i++) {
cols[i] = i;
vals[i] = 0.0;
}
ierr = MatGetOwnershipRange(*J,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatSetValues(*J,1,&i,red->N,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree2(cols,vals);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
