/*@
MatXAIJSetPreallocation - set preallocation for serial and parallel AIJ, BAIJ, and SBAIJ matrices
Collective on Mat
Input Arguments:
+ A - matrix being preallocated
. bs - block size
. dnnz - number of nonzero blocks per block row of diagonal part of parallel matrix
. onnz - number of nonzero blocks per block row of off-diagonal part of parallel matrix
. dnnzu - number of nonzero blocks per block row of upper-triangular part of diagonal part of parallel matrix
- onnzu - number of nonzero blocks per block row of upper-triangular part of off-diagonal part of parallel matrix
Level: beginner
.seealso: MatSeqAIJSetPreallocation(), MatMPIAIJSetPreallocation(), MatSeqBAIJSetPreallocation(), MatMPIBAIJSetPreallocation(), MatSeqSBAIJSetPreallocation(), MatMPISBAIJSetPreallocation(),
PetscSplitOwnership()
@*/
PetscErrorCode MatXAIJSetPreallocation(Mat A,PetscInt bs,const PetscInt dnnz[],const PetscInt onnz[],const PetscInt dnnzu[],const PetscInt onnzu[])
{
PetscErrorCode ierr;
void (*aij)(void);
PetscFunctionBegin;
ierr = MatSetBlockSize(A,bs);CHKERRQ(ierr);
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->cmap);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,0,dnnz);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,0,dnnz,0,onnz);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(A,bs,0,dnnzu);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(A,bs,0,dnnzu,0,onnzu);CHKERRQ(ierr);
/*
In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
good before going on with it.
*/
ierr = PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);CHKERRQ(ierr);
if (!aij) {
ierr = PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);CHKERRQ(ierr);
}
if (aij) {
if (bs == 1) {
ierr = MatSeqAIJSetPreallocation(A,0,dnnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz,0,onnz);CHKERRQ(ierr);
} else { /* Convert block-row precallocation to scalar-row */
PetscInt i,m,*sdnnz,*sonnz;
ierr = MatGetLocalSize(A,&m,NULL);CHKERRQ(ierr);
ierr = PetscMalloc2((!!dnnz)*m,&sdnnz,(!!onnz)*m,&sonnz);CHKERRQ(ierr);
for (i=0; i<m; i++) {
if (dnnz) sdnnz[i] = dnnz[i/bs] * bs;
if (onnz) sonnz[i] = onnz[i/bs] * bs;
}
ierr = MatSeqAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,0,dnnz ? sdnnz : NULL,0,onnz ? sonnz : NULL);CHKERRQ(ierr);
ierr = PetscFree2(sdnnz,sonnz);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscInt bs,i,nlocalcols;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
ierr = MatISComputeSF_Private(B);CHKERRQ(ierr);
}
if (!d_nnz) {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nz;
} else {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] = d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nz;
} else {
for (i=0;i<matis->sf_nroots;i++) matis->sf_rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,NULL,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(matis->sf,MPIU_INT,matis->sf_rootdata,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves;i++) {
matis->sf_leafdata[i] = PetscMin(matis->sf_leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves/bs;i++) {
matis->sf_leafdata[i] = matis->sf_leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);CHKERRQ(ierr);
for (i=0;i<matis->sf_nleaves/bs;i++) {
matis->sf_leafdata[i] = matis->sf_leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,matis->sf_leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode DMCreateMatrix_Redundant(DM dm,Mat *J)
{
DM_Redundant *red = (DM_Redundant*)dm->data;
PetscErrorCode ierr;
ISLocalToGlobalMapping ltog,ltogb;
PetscInt i,rstart,rend,*cols;
PetscScalar *vals;
PetscFunctionBegin;
ierr = MatCreate(PetscObjectComm((PetscObject)dm),J);CHKERRQ(ierr);
ierr = MatSetSizes(*J,red->n,red->n,red->N,red->N);CHKERRQ(ierr);
ierr = MatSetType(*J,dm->mattype);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*J,red->n,NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(*J,1,red->n,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,red->n,NULL,red->N-red->n,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(*J,1,red->n,NULL,red->N-red->n,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(dm,<og);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMappingBlock(dm,<ogb);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(*J,ltog,ltog);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMappingBlock(*J,ltogb,ltogb);CHKERRQ(ierr);
ierr = PetscMalloc2(red->N,&cols,red->N,&vals);CHKERRQ(ierr);
for (i=0; i<red->N; i++) {
cols[i] = i;
vals[i] = 0.0;
}
ierr = MatGetOwnershipRange(*J,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
ierr = MatSetValues(*J,1,&i,red->N,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree2(cols,vals);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
EXTERN_C_END
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "MatConvert_SeqSBAI_SeqBAIJ"
PetscErrorCode PETSCMAT_DLLEXPORT MatConvert_SeqSBAIJ_SeqBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat B;
Mat_SeqSBAIJ *a = (Mat_SeqSBAIJ*)A->data;
Mat_SeqBAIJ *b;
PetscErrorCode ierr;
PetscInt *ai=a->i,*aj=a->j,m=A->rmap->N,n=A->cmap->n,i,k,*bi,*bj,*browlengths,nz,*browstart,itmp;
PetscInt bs=A->rmap->bs,bs2=bs*bs,mbs=m/bs;
MatScalar *av,*bv;
PetscFunctionBegin;
/* compute browlengths of newmat */
ierr = PetscMalloc2(mbs,PetscInt,&browlengths,mbs,PetscInt,&browstart);CHKERRQ(ierr);
for (i=0; i<mbs; i++) browlengths[i] = 0;
aj = a->j;
for (i=0; i<mbs; i++) {
nz = ai[i+1] - ai[i];
aj++; /* skip diagonal */
for (k=1; k<nz; k++) { /* no. of lower triangular blocks */
browlengths[*aj]++; aj++;
}
browlengths[i] += nz; /* no. of upper triangular blocks */
}
ierr = MatCreate(((PetscObject)A)->comm,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,m,n,m,n);CHKERRQ(ierr);
ierr = MatSetType(B,newtype);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(B,bs,0,browlengths);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_ROW_ORIENTED,PETSC_TRUE);CHKERRQ(ierr);
b = (Mat_SeqBAIJ*)(B->data);
bi = b->i;
bj = b->j;
bv = b->a;
/* set b->i */
bi[0] = 0;
for (i=0; i<mbs; i++){
b->ilen[i] = browlengths[i];
bi[i+1] = bi[i] + browlengths[i];
browstart[i] = bi[i];
}
if (bi[mbs] != 2*a->nz - mbs) SETERRQ2(PETSC_ERR_PLIB,"bi[mbs]: %D != 2*a->nz - mbs: %D\n",bi[mbs],2*a->nz - mbs);
/* set b->j and b->a */
aj = a->j; av = a->a;
for (i=0; i<mbs; i++) {
/* diagonal block */
*(bj + browstart[i]) = *aj; aj++;
itmp = bs2*browstart[i];
for (k=0; k<bs2; k++){
*(bv + itmp + k) = *av; av++;
}
browstart[i]++;
nz = ai[i+1] - ai[i] -1;
while (nz--){
/* lower triangular blocks */
*(bj + browstart[*aj]) = i;
itmp = bs2*browstart[*aj];
for (k=0; k<bs2; k++){
*(bv + itmp + k) = *(av + k);
}
browstart[*aj]++;
/* upper triangular blocks */
*(bj + browstart[i]) = *aj; aj++;
itmp = bs2*browstart[i];
for (k=0; k<bs2; k++){
*(bv + itmp + k) = *av; av++;
}
browstart[i]++;
}
}
ierr = PetscFree2(browlengths,browstart);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_REUSE_MATRIX) {
ierr = MatHeaderReplace(A,B);CHKERRQ(ierr);
} else {
*newmat = B;
}
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,Atrans,sA,*submatA,*submatsA;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt bs=1,mbs=10,ov=1,i,j,k,*rows,*cols,nd=2,*idx,rstart,rend,sz,M,N,Mbs;
PetscScalar *vals,rval,one=1.0;
IS *is1,*is2;
PetscRandom rand;
PetscBool flg,TestOverlap,TestSubMat,TestAllcols,test_sorted=PETSC_FALSE;
PetscInt vid = -1;
#if defined(PETSC_USE_LOG)
PetscLogStage stages[2];
#endif
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_mbs",&mbs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-view_id",&vid,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_overlap", &TestOverlap);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_submat", &TestSubMat);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_allcols", &TestAllcols);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-test_sorted",&test_sorted,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,mbs*bs,mbs*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (j=0; j<bs*bs; j++) vals[j] = 0.0;
for (i=0; i<Mbs; i++) {
cols[0] = i*bs; rows[0] = i*bs;
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
/* second, add random blocks */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
cols[0] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(PetscInt)(PetscRealPart(rval)*mbs);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* make A a symmetric matrix: A <- A^T + A */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatEqual(A, Atrans, &flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric");
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* create a SeqSBAIJ matrix sA (= A) */
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
if (vid >= 0 && vid < size) {
if (!rank) printf("A: \n");
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (!rank) printf("sA: \n");
ierr = MatView(sA,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Test sA==A through MatMult() */
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Error in MatConvert(): A != sA");
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
for (i=0; i<nd; i++) {
if (!TestAllcols) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
sz = (PetscInt)((0.5+0.2*PetscRealPart(rval))*mbs); /* 0.5*mbs < sz < 0.7*mbs */
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
if (rank == vid) {
ierr = PetscPrintf(PETSC_COMM_SELF," [%d] IS sz[%d]: %d\n",rank,i,sz);CHKERRQ(ierr);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
} else { /* Test all rows and colums */
sz = M;
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is1+i);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is2+i);CHKERRQ(ierr);
if (rank == vid) {
PetscBool colflag;
ierr = ISIdentity(is2[i],&colflag);CHKERRQ(ierr);
printf("[%d] is2[%d], colflag %d\n",rank,(int)i,(int)colflag);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
}
}
ierr = PetscLogStageRegister("MatOv_SBAIJ",&stages[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("MatOv_BAIJ",&stages[1]);CHKERRQ(ierr);
/* Test MatIncreaseOverlap */
if (TestOverlap) {
ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (rank == vid) {
printf("\n[%d] IS from BAIJ:\n",rank);
ierr = ISView(is1[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
printf("\n[%d] IS from SBAIJ:\n",rank);
ierr = ISView(is2[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
if (!rank) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
/* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
ierr = ISSort(is2[i]);CHKERRQ(ierr);
/* ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
}
SETERRQ1(PETSC_COMM_SELF,1,"i=%D, is1 != is2",i);
}
}
}
/* Test MatCreateSubmatrices */
if (TestSubMat) {
if (test_sorted) {
for (i = 0; i < nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
}
}
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"A != sA");
/* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"MatCreateSubmatrices(): A != sA");
ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr);
ierr = MatDestroySubMatrices(nd,&submatsA);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@C
SlicedGetMatrix - Creates a matrix with the correct parallel layout required for
computing the Jacobian on a function defined using the informatin in Sliced.
Collective on Sliced
Input Parameter:
+ slice - the slice object
- mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, MATSEQSBAIJ, MATMPISBAIJ,
or any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).
Output Parameters:
. J - matrix with the correct nonzero preallocation
(obviously without the correct Jacobian values)
Level: advanced
Notes: This properly preallocates the number of nonzeros in the sparse matrix so you
do not need to do it yourself.
.seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), DASetBlockFills()
@*/
PetscErrorCode PETSCDM_DLLEXPORT SlicedGetMatrix(Sliced slice, const MatType mtype,Mat *J)
{
PetscErrorCode ierr;
PetscInt *globals,*sd_nnz,*so_nnz,rstart,bs,i;
ISLocalToGlobalMapping lmap,blmap;
void (*aij)(void) = PETSC_NULL;
PetscFunctionBegin;
bs = slice->bs;
ierr = MatCreate(((PetscObject)slice)->comm,J);CHKERRQ(ierr);
ierr = MatSetSizes(*J,slice->n*bs,slice->n*bs,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(*J,mtype);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(*J,bs,slice->d_nz,slice->d_nnz);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(*J,bs,slice->d_nz,slice->d_nnz,slice->o_nz,slice->o_nnz);CHKERRQ(ierr);
/* In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
* good before going on with it. */
ierr = PetscObjectQueryFunction((PetscObject)*J,"MatMPIAIJSetPreallocation_C",&aij);CHKERRQ(ierr);
if (!aij) {
ierr = PetscObjectQueryFunction((PetscObject)*J,"MatSeqAIJSetPreallocation_C",&aij);CHKERRQ(ierr);
}
if (aij) {
if (bs == 1) {
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz,slice->d_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz,slice->d_nnz,slice->o_nz,slice->o_nnz);CHKERRQ(ierr);
} else if (!slice->d_nnz) {
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz*bs,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz*bs,PETSC_NULL,slice->o_nz*bs,PETSC_NULL);CHKERRQ(ierr);
} else {
/* The user has provided preallocation per block-row, convert it to per scalar-row respecting SlicedSetBlockFills() if applicable */
ierr = PetscMalloc2(slice->n*bs,PetscInt,&sd_nnz,(!!slice->o_nnz)*slice->n*bs,PetscInt,&so_nnz);CHKERRQ(ierr);
for (i=0; i<slice->n*bs; i++) {
sd_nnz[i] = (slice->d_nnz[i/bs]-1) * (slice->ofill ? slice->ofill->i[i%bs+1]-slice->ofill->i[i%bs] : bs)
+ (slice->dfill ? slice->dfill->i[i%bs+1]-slice->dfill->i[i%bs] : bs);
if (so_nnz) {
so_nnz[i] = slice->o_nnz[i/bs] * (slice->ofill ? slice->ofill->i[i%bs+1]-slice->ofill->i[i%bs] : bs);
}
}
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz*bs,sd_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz*bs,sd_nnz,slice->o_nz*bs,so_nnz);CHKERRQ(ierr);
ierr = PetscFree2(sd_nnz,so_nnz);CHKERRQ(ierr);
}
}
ierr = MatSetBlockSize(*J,bs);CHKERRQ(ierr);
/* Set up the local to global map. For the scalar map, we have to translate to entry-wise indexing instead of block-wise. */
ierr = PetscMalloc((slice->n+slice->Nghosts)*bs*sizeof(PetscInt),&globals);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(*J,&rstart,PETSC_NULL);CHKERRQ(ierr);
for (i=0; i<slice->n*bs; i++) {
globals[i] = rstart + i;
}
for (i=0; i<slice->Nghosts*bs; i++) {
globals[slice->n*bs+i] = slice->ghosts[i/bs]*bs + i%bs;
}
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,(slice->n+slice->Nghosts)*bs,globals,&lmap);CHKERRQ(ierr);
ierr = PetscFree(globals);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingBlock(lmap,bs,&blmap);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(*J,lmap);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMappingBlock(*J,blmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(lmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(blmap);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int
PetscSparseMtrx :: buildInternalStructure(EngngModel *eModel, int di, EquationID ut, const UnknownNumberingScheme &s)
{
IntArray loc;
Domain *domain = eModel->giveDomain(di);
int nelem;
if ( mtrx ) {
MatDestroy(&mtrx);
}
if ( this->kspInit ) {
KSPDestroy(&ksp);
this->kspInit = false; // force ksp to be initialized
}
this->ut = ut;
this->emodel = eModel;
this->di = di;
#ifdef __PARALLEL_MODE
if ( eModel->isParallel() ) {
int rank;
PetscNatural2GlobalOrdering *n2g;
PetscNatural2LocalOrdering *n2l;
rank = eModel->giveRank();
n2g = eModel->givePetscContext(di, ut)->giveN2Gmap();
n2l = eModel->givePetscContext(di, ut)->giveN2Lmap();
n2l->init(eModel, ut, di);
n2g->init(eModel, ut, di);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscSparseMtrx:: buildInternalStructure", "", rank);
#endif
leqs = n2g->giveNumberOfLocalEqs();
geqs = n2g->giveNumberOfGlobalEqs();
//printf("%d, leqs = %d, geqs = %d\n", this->emodel->giveRank(), leqs, geqs);
#ifdef __VERBOSE_PARALLEL
OOFEM_LOG_INFO( "[%d]PetscSparseMtrx:: buildInternalStructure: l_eqs = %d, g_eqs = %d, n_eqs = %d\n", rank, leqs, geqs, eModel->giveNumberOfEquations(ut) );
#endif
// determine nonzero structure of a "local (maintained)" part of matrix, and the off-diagonal part
int i, ii, j, jj, n;
Element *elem;
// allocation map
std :: vector< std :: set< int > >d_rows(leqs); // diagonal sub-matrix allocation
std :: vector< std :: set< int > >o_rows(leqs); // off-diagonal allocation
IntArray d_nnz(leqs), o_nnz(leqs), lloc, gloc;
//fprintf (stderr,"[%d] n2l map: ",rank);
//for (n=1; n<=n2l.giveN2Lmap()->giveSize(); n++) fprintf (stderr, "%d ", n2l.giveN2Lmap()->at(n));
nelem = domain->giveNumberOfElements();
for ( n = 1; n <= nelem; n++ ) {
//fprintf (stderr, "(elem %d) ", n);
elem = domain->giveElement(n);
elem->giveLocationArray(loc, ut, s);
n2l->map2New(lloc, loc, 0); // translate natural->local numbering (remark, 1-based indexing)
n2g->map2New(gloc, loc, 0); // translate natural->global numbering (remark, 0-based indexing)
// See the petsc manual for details on how this allocation is constructed.
for ( i = 1; i <= lloc.giveSize(); i++ ) {
if ( ( ii = lloc.at(i) ) ) {
for ( j = 1; j <= lloc.giveSize(); j++ ) {
if ( ( jj = gloc.at(j) ) >= 0 ) { // if negative, then it is prescribed
if ( lloc.at(j) ) { // if true, then its the local part (the diagonal block matrix)
d_rows [ ii - 1 ].insert(jj);
} else { // Otherwise it must be off-diagonal
o_rows [ ii - 1 ].insert(jj);
}
}
}
}
}
//fprintf (stderr, "\n");
}
// Diagonal must always be allocated; this code ensures that for every local line, it adds the global column number
IntArray *n2gmap = n2g->giveN2Gmap();
IntArray *n2lmap = n2l->giveN2Lmap();
for ( int n = 1; n <= n2lmap->giveSize(); ++n ) {
if ( n2lmap->at(n) ) {
d_rows [ n2lmap->at(n)-1 ].insert( n2gmap->at(n) );
}
}
for ( i = 0; i < leqs; i++ ) {
d_nnz(i) = d_rows [ i ].size();
o_nnz(i) = o_rows [ i ].size();
}
//fprintf (stderr,"\n[%d]PetscSparseMtrx: Profile ...",rank);
//for (i=0; i<leqs; i++) fprintf(stderr, "%d ", d_nnz(i));
//fprintf (stderr,"\n[%d]PetscSparseMtrx: Creating MPIAIJ Matrix ...\n",rank);
// create PETSc mat
MatCreate(PETSC_COMM_WORLD, & mtrx);
MatSetSizes(mtrx, leqs, leqs, geqs, geqs);
MatSetType(mtrx, MATMPIAIJ);
MatSetFromOptions(mtrx);
MatSetUp(mtrx);
MatMPIAIJSetPreallocation(mtrx, 0, d_nnz.givePointer(), 0, o_nnz.givePointer());
//MatMPIBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz.givePointer(), onz, onnz );
//MatMPISBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz_sym.givePointer(), onz, onnz );
MatSetOption(mtrx, MAT_ROW_ORIENTED, PETSC_FALSE); // To allow the insertion of values using MatSetValues in column major order
MatSetOption(mtrx, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
#ifdef __VERBOSE_PARALLEL
VERBOSEPARALLEL_PRINT("PetscSparseMtrx:: buildInternalStructure", "done", rank);
#endif
} else {
#endif
leqs = geqs = eModel->giveNumberOfEquations(ut);
int i, ii, j, jj, n;
Element *elem;
// allocation map
std :: vector< std :: set< int > >rows(leqs);
std :: vector< std :: set< int > >rows_sym(leqs);
nelem = domain->giveNumberOfElements();
for ( n = 1; n <= nelem; n++ ) {
elem = domain->giveElement(n);
elem->giveLocationArray(loc, ut, s);
for ( i = 1; i <= loc.giveSize(); i++ ) {
if ( ( ii = loc.at(i) ) ) {
for ( j = 1; j <= loc.giveSize(); j++ ) {
jj = loc.at(j);
if ( jj ) {
rows [ ii - 1 ].insert(jj - 1);
if ( jj >= ii ) {
rows_sym [ ii - 1 ].insert(jj - 1);
}
}
}
}
}
}
// Structure from active boundary conditions.
AList<IntArray> locs, temp;
for ( n = 1; n <= domain->giveNumberOfBoundaryConditions(); n++ ) {
ActiveBoundaryCondition *activebc = dynamic_cast<ActiveBoundaryCondition*>(domain->giveBc(n));
if (activebc) {
///@todo Deal with the CharType here.
activebc->giveLocationArrays(locs, temp, ut, TangentStiffnessMatrix, s, s, domain);
for (int k = 1; k < locs.giveSize(); k++) {
IntArray *kloc = locs.at(k);
for ( i = 1; i <= kloc->giveSize(); i++ ) {
if ( ( ii = kloc->at(i) ) ) {
for ( j = 1; j <= kloc->giveSize(); j++ ) {
jj = kloc->at(j);
if ( jj ) {
rows [ ii - 1 ].insert(jj - 1);
if ( jj >= ii ) {
rows_sym [ ii - 1 ].insert(jj - 1);
}
}
}
}
}
}
}
}
IntArray d_nnz(leqs);
IntArray d_nnz_sym(leqs);
for ( i = 0; i < leqs; i++ ) {
d_nnz(i) = rows [ i ].size();
d_nnz_sym(i) = rows_sym [ i ].size();
}
MatCreate(PETSC_COMM_SELF, & mtrx);
MatSetSizes(mtrx, leqs, leqs, geqs, geqs);
MatSetType(mtrx, MATSEQAIJ);
//MatSetType(mtrx, MATSBAIJ);
//MatSetType(mtrx, MATDENSE);
MatSetFromOptions(mtrx);
MatSetUp(mtrx);
MatSeqAIJSetPreallocation( mtrx, 0, d_nnz.givePointer() );
MatSeqBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz.givePointer() );
MatSeqSBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz_sym.givePointer() );
MatSetOption(mtrx, MAT_ROW_ORIENTED, PETSC_FALSE); // To allow the insertion of values using MatSetValues in column major order
MatSetOption(mtrx, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
#ifdef __PARALLEL_MODE
}
#endif
nRows = nColumns = geqs;
this->newValues = true;
return true;
}
int main(int argc,char **args)
{
Mat C,A;
PetscInt i, n = 10,midx[3],bs=1;
PetscErrorCode ierr;
PetscScalar v[3];
PetscBool flg,isAIJ;
MatType type;
PetscMPIInt size;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_block_size",&bs,PETSC_NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetType(C,MATAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatGetType(C,&type);CHKERRQ(ierr);
if (size == 1){
ierr = PetscObjectTypeCompare((PetscObject)C,MATSEQAIJ,&isAIJ);CHKERRQ(ierr);
} else {
ierr = PetscObjectTypeCompare((PetscObject)C,MATMPIAIJ,&isAIJ);CHKERRQ(ierr);
}
ierr = MatSeqAIJSetPreallocation(C,3,PETSC_NULL);
ierr = MatMPIAIJSetPreallocation(C,3,PETSC_NULL,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(C,bs,3,PETSC_NULL);
ierr = MatMPIBAIJSetPreallocation(C,bs,3,PETSC_NULL,3,PETSC_NULL);CHKERRQ(ierr);
v[0] = -1.; v[1] = 2.; v[2] = -1.;
for (i=1; i<n-1; i++){
midx[2] = i-1; midx[1] = i; midx[0] = i+1;
ierr = MatSetValues(C,1,&i,3,midx,v,INSERT_VALUES);CHKERRQ(ierr);
}
i = 0; midx[0] = 0; midx[1] = 1;
v[0] = 2.0; v[1] = -1.;
ierr = MatSetValues(C,1,&i,2,midx,v,INSERT_VALUES);CHKERRQ(ierr);
i = n-1; midx[0] = n-2; midx[1] = n-1;
v[0] = -1.0; v[1] = 2.;
ierr = MatSetValues(C,1,&i,2,midx,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* test matrices with different nonzero patterns - Note: A is created with different nonzero pattern of C! */
ierr = MatCopy(C,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatEqual(A,C,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"MatCopy(C,A,DIFFERENT_NONZERO_PATTERN): Matrices are NOT equal");
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"A is obtained with MatCopy(,,DIFFERENT_NONZERO_PATTERN):\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
/* test matrices with same nonzero pattern */
ierr = MatDuplicate(C,MAT_DO_NOT_COPY_VALUES,&A);CHKERRQ(ierr);
ierr = MatCopy(C,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatEqual(A,C,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"MatCopy(C,A,SAME_NONZERO_PATTERN): Matrices are NOT equal");
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nA is obtained with MatCopy(,,SAME_NONZERO_PATTERN):\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_COMMON);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"A:\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test MatStore/RetrieveValues() */
if (isAIJ){
ierr = MatSetOption(A,MAT_NEW_NONZERO_LOCATIONS,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatStoreValues(A);CHKERRQ(ierr);
ierr = MatZeroEntries(A);CHKERRQ(ierr);
ierr = MatRetrieveValues(A);CHKERRQ(ierr);
}
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void PETSC_STDCALL matseqbaijsetpreallocation_(Mat *mat,PetscInt *bs,PetscInt *nz,PetscInt *nnz,PetscErrorCode *ierr)
{
CHKFORTRANNULLINTEGER(nnz);
*ierr = MatSeqBAIJSetPreallocation(*mat,*bs,*nz,nnz);
}
int main(int argc,char **args)
{
Mat A,B,*submatA,*submatB;
PetscInt bs=1,m=11,ov=1,i,j,k,*rows,*cols,nd=5,*idx,rstart,rend,sz,mm,nn,M,N,Mbs;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscScalar *vals,rval;
IS *is1,*is2;
PetscRandom rdm;
Vec xx,s1,s2;
PetscReal s1norm,s2norm,rnorm,tol = 100*PETSC_SMALL;
PetscBool flg,test_nd0=PETSC_FALSE;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_size",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-test_nd0",&test_nd0,NULL);CHKERRQ(ierr);
/* Create a AIJ matrix A */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m*bs,m*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
/* Create a BAIJ matrix B */
ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,m*bs,m*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(B,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(B,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(B,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (i=0; i<40*bs; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
cols[0] = bs*(int)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(int)(PetscRealPart(rval)*m);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(B,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
if (!rank && test_nd0) nd = 0; /* test case */
for (i=0; i<nd; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
sz = (int)(PetscRealPart(rval)*m);
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(int)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
}
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(B,nd,is2,ov);CHKERRQ(ierr);
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
ierr = PetscPrintf(PETSC_COMM_SELF,"i=%D, flg=%d :bs=%D m=%D ov=%D nd=%D np=%D\n",i,flg,bs,m,ov,nd,size);CHKERRQ(ierr);
}
}
for (i=0; i<nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
ierr = ISSort(is2[i]);CHKERRQ(ierr);
}
ierr = MatCreateSubMatrices(B,nd,is2,is2,MAT_INITIAL_MATRIX,&submatB);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatB[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d]Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",rank,s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(B,nd,is2,is2,MAT_REUSE_MATRIX,&submatB);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatB[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d]Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",rank,s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr);
ierr = MatDestroySubMatrices(nd,&submatB);CHKERRQ(ierr);
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
void PetscMatrix<T>::init ()
{
libmesh_assert(this->_dof_map);
// Clear initialized matrices
if (this->initialized())
this->clear();
this->_is_initialized = true;
const numeric_index_type my_m = this->_dof_map->n_dofs();
const numeric_index_type my_n = my_m;
const numeric_index_type n_l = this->_dof_map->n_dofs_on_processor(this->processor_id());
const numeric_index_type m_l = n_l;
const std::vector<numeric_index_type>& n_nz = this->_dof_map->get_n_nz();
const std::vector<numeric_index_type>& n_oz = this->_dof_map->get_n_oz();
// Make sure the sparsity pattern isn't empty unless the matrix is 0x0
libmesh_assert_equal_to (n_nz.size(), m_l);
libmesh_assert_equal_to (n_oz.size(), m_l);
PetscErrorCode ierr = 0;
PetscInt m_global = static_cast<PetscInt>(my_m);
PetscInt n_global = static_cast<PetscInt>(my_n);
PetscInt m_local = static_cast<PetscInt>(m_l);
PetscInt n_local = static_cast<PetscInt>(n_l);
ierr = MatCreate(this->comm().get(), &_mat);
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetSizes(_mat, m_local, n_local, m_global, n_global);
LIBMESH_CHKERRABORT(ierr);
#ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
PetscInt blocksize = static_cast<PetscInt>(this->_dof_map->block_size());
if (blocksize > 1)
{
// specified blocksize, bs>1.
// double check sizes.
libmesh_assert_equal_to (m_local % blocksize, 0);
libmesh_assert_equal_to (n_local % blocksize, 0);
libmesh_assert_equal_to (m_global % blocksize, 0);
libmesh_assert_equal_to (n_global % blocksize, 0);
ierr = MatSetType(_mat, MATBAIJ); // Automatically chooses seqbaij or mpibaij
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetBlockSize(_mat, blocksize);
LIBMESH_CHKERRABORT(ierr);
// transform the per-entry n_nz and n_oz arrays into their block counterparts.
std::vector<numeric_index_type> b_n_nz, b_n_oz;
transform_preallocation_arrays (blocksize,
n_nz, n_oz,
b_n_nz, b_n_oz);
ierr = MatSeqBAIJSetPreallocation(_mat, blocksize, 0, (PetscInt*)(b_n_nz.empty()?NULL:&b_n_nz[0]));
LIBMESH_CHKERRABORT(ierr);
ierr = MatMPIBAIJSetPreallocation(_mat, blocksize,
0, (PetscInt*)(b_n_nz.empty()?NULL:&b_n_nz[0]),
0, (PetscInt*)(b_n_oz.empty()?NULL:&b_n_oz[0]));
LIBMESH_CHKERRABORT(ierr);
}
else
#endif
{
// no block storage case
ierr = MatSetType(_mat, MATAIJ); // Automatically chooses seqaij or mpiaij
LIBMESH_CHKERRABORT(ierr);
ierr = MatSeqAIJSetPreallocation(_mat, 0, (PetscInt*)(n_nz.empty()?NULL:&n_nz[0]));
LIBMESH_CHKERRABORT(ierr);
ierr = MatMPIAIJSetPreallocation(_mat,
0, (PetscInt*)(n_nz.empty()?NULL:&n_nz[0]),
0, (PetscInt*)(n_oz.empty()?NULL:&n_oz[0]));
LIBMESH_CHKERRABORT(ierr);
}
// Is prefix information available somewhere? Perhaps pass in the system name?
ierr = MatSetOptionsPrefix(_mat, "");
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetFromOptions(_mat);
LIBMESH_CHKERRABORT(ierr);
this->zero();
}
void PetscMatrix<T>::init (const numeric_index_type m_in,
const numeric_index_type n_in,
const numeric_index_type m_l,
const numeric_index_type n_l,
const numeric_index_type nnz,
const numeric_index_type noz,
const numeric_index_type blocksize_in)
{
// So compilers don't warn when !LIBMESH_ENABLE_BLOCKED_STORAGE
libmesh_ignore(blocksize_in);
// Clear initialized matrices
if (this->initialized())
this->clear();
this->_is_initialized = true;
PetscErrorCode ierr = 0;
PetscInt m_global = static_cast<PetscInt>(m_in);
PetscInt n_global = static_cast<PetscInt>(n_in);
PetscInt m_local = static_cast<PetscInt>(m_l);
PetscInt n_local = static_cast<PetscInt>(n_l);
PetscInt n_nz = static_cast<PetscInt>(nnz);
PetscInt n_oz = static_cast<PetscInt>(noz);
ierr = MatCreate(this->comm().get(), &_mat);
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetSizes(_mat, m_local, n_local, m_global, n_global);
LIBMESH_CHKERRABORT(ierr);
#ifdef LIBMESH_ENABLE_BLOCKED_STORAGE
PetscInt blocksize = static_cast<PetscInt>(blocksize_in);
if (blocksize > 1)
{
// specified blocksize, bs>1.
// double check sizes.
libmesh_assert_equal_to (m_local % blocksize, 0);
libmesh_assert_equal_to (n_local % blocksize, 0);
libmesh_assert_equal_to (m_global % blocksize, 0);
libmesh_assert_equal_to (n_global % blocksize, 0);
libmesh_assert_equal_to (n_nz % blocksize, 0);
libmesh_assert_equal_to (n_oz % blocksize, 0);
ierr = MatSetType(_mat, MATBAIJ); // Automatically chooses seqbaij or mpibaij
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetBlockSize(_mat, blocksize);
LIBMESH_CHKERRABORT(ierr);
ierr = MatSeqBAIJSetPreallocation(_mat, blocksize, n_nz/blocksize, PETSC_NULL);
LIBMESH_CHKERRABORT(ierr);
ierr = MatMPIBAIJSetPreallocation(_mat, blocksize,
n_nz/blocksize, PETSC_NULL,
n_oz/blocksize, PETSC_NULL);
LIBMESH_CHKERRABORT(ierr);
}
else
#endif
{
ierr = MatSetType(_mat, MATAIJ); // Automatically chooses seqaij or mpiaij
LIBMESH_CHKERRABORT(ierr);
ierr = MatSeqAIJSetPreallocation(_mat, n_nz, PETSC_NULL);
LIBMESH_CHKERRABORT(ierr);
ierr = MatMPIAIJSetPreallocation(_mat, n_nz, PETSC_NULL, n_oz, PETSC_NULL);
LIBMESH_CHKERRABORT(ierr);
}
// Make it an error for PETSc to allocate new nonzero entries during assembly
#if PETSC_VERSION_LESS_THAN(3,0,0)
ierr = MatSetOption(_mat, MAT_NEW_NONZERO_ALLOCATION_ERR);
#else
ierr = MatSetOption(_mat, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE);
#endif
LIBMESH_CHKERRABORT(ierr);
// Is prefix information available somewhere? Perhaps pass in the system name?
ierr = MatSetOptionsPrefix(_mat, "");
LIBMESH_CHKERRABORT(ierr);
ierr = MatSetFromOptions(_mat);
LIBMESH_CHKERRABORT(ierr);
this->zero ();
}
/*
Takes the local part of an already assembled MPISBAIJ matrix
and disassembles it. This is to allow new nonzeros into the matrix
that require more communication in the matrix vector multiply.
Thus certain data-structures must be rebuilt.
Kind of slow! But that's what application programmers get when
they are sloppy.
*/
PetscErrorCode MatDisAssemble_MPISBAIJ(Mat A)
{
Mat_MPISBAIJ *baij = (Mat_MPISBAIJ*)A->data;
Mat B = baij->B,Bnew;
Mat_SeqBAIJ *Bbaij = (Mat_SeqBAIJ*)B->data;
PetscErrorCode ierr;
PetscInt i,j,mbs=Bbaij->mbs,n = A->cmap->N,col,*garray=baij->garray;
PetscInt k,bs=A->rmap->bs,bs2=baij->bs2,*rvals,*nz,ec,m=A->rmap->n;
MatScalar *a = Bbaij->a;
PetscScalar *atmp;
#if defined(PETSC_USE_REAL_MAT_SINGLE)
PetscInt l;
#endif
PetscFunctionBegin;
#if defined(PETSC_USE_REAL_MAT_SINGLE)
ierr = PetscMalloc1(A->rmap->bs,&atmp);
#endif
/* free stuff related to matrix-vec multiply */
ierr = VecGetSize(baij->lvec,&ec);CHKERRQ(ierr); /* needed for PetscLogObjectMemory below */
ierr = VecDestroy(&baij->lvec);CHKERRQ(ierr);
ierr = VecScatterDestroy(&baij->Mvctx);CHKERRQ(ierr);
ierr = VecDestroy(&baij->slvec0);CHKERRQ(ierr);
ierr = VecDestroy(&baij->slvec0b);CHKERRQ(ierr);
ierr = VecDestroy(&baij->slvec1);CHKERRQ(ierr);
ierr = VecDestroy(&baij->slvec1a);CHKERRQ(ierr);
ierr = VecDestroy(&baij->slvec1b);CHKERRQ(ierr);
if (baij->colmap) {
#if defined(PETSC_USE_CTABLE)
ierr = PetscTableDestroy(&baij->colmap);CHKERRQ(ierr);
#else
ierr = PetscFree(baij->colmap);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)A,-Bbaij->nbs*sizeof(PetscInt));CHKERRQ(ierr);
#endif
}
/* make sure that B is assembled so we can access its values */
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* invent new B and copy stuff over */
ierr = PetscMalloc1(mbs,&nz);CHKERRQ(ierr);
for (i=0; i<mbs; i++) {
nz[i] = Bbaij->i[i+1]-Bbaij->i[i];
}
ierr = MatCreate(PETSC_COMM_SELF,&Bnew);CHKERRQ(ierr);
ierr = MatSetSizes(Bnew,m,n,m,n);CHKERRQ(ierr);
ierr = MatSetType(Bnew,((PetscObject)B)->type_name);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(Bnew,B->rmap->bs,0,nz);CHKERRQ(ierr);
ierr = PetscFree(nz);CHKERRQ(ierr);
if (Bbaij->nonew >= 0) { /* Inherit insertion error options (if positive). */
((Mat_SeqSBAIJ*)Bnew->data)->nonew = Bbaij->nonew;
}
/*
Ensure that B's nonzerostate is monotonically increasing.
Or should this follow the MatSetValues() loop to preserve B's nonzerstate across a MatDisAssemble() call?
*/
Bnew->nonzerostate = B->nonzerostate;
ierr = PetscMalloc1(bs,&rvals);CHKERRQ(ierr);
for (i=0; i<mbs; i++) {
rvals[0] = bs*i;
for (j=1; j<bs; j++) rvals[j] = rvals[j-1] + 1;
for (j=Bbaij->i[i]; j<Bbaij->i[i+1]; j++) {
col = garray[Bbaij->j[j]]*bs;
for (k=0; k<bs; k++) {
#if defined(PETSC_USE_REAL_MAT_SINGLE)
for (l=0; l<bs; l++) atmp[l] = a[j*bs2+l];
#else
atmp = a+j*bs2 + k*bs;
#endif
ierr = MatSetValues_SeqSBAIJ(Bnew,bs,rvals,1,&col,atmp,B->insertmode);CHKERRQ(ierr);
col++;
}
}
}
#if defined(PETSC_USE_REAL_MAT_SINGLE)
ierr = PetscFree(atmp);CHKERRQ(ierr);
#endif
ierr = PetscFree(baij->garray);CHKERRQ(ierr);
baij->garray = 0;
ierr = PetscFree(rvals);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)A,-ec*sizeof(PetscInt));CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)A,(PetscObject)Bnew);CHKERRQ(ierr);
baij->B = Bnew;
A->was_assembled = PETSC_FALSE;
PetscFunctionReturn(0);
}
PetscErrorCode MatLUFactorSymbolic_SeqBAIJ_inplace(Mat B,Mat A,IS isrow,IS iscol,const MatFactorInfo *info)
{
Mat_SeqBAIJ *a = (Mat_SeqBAIJ*)A->data,*b;
PetscInt n =a->mbs,bs = A->rmap->bs,bs2=a->bs2;
PetscBool row_identity,col_identity,both_identity;
IS isicol;
PetscErrorCode ierr;
const PetscInt *r,*ic;
PetscInt i,*ai=a->i,*aj=a->j;
PetscInt *bi,*bj,*ajtmp;
PetscInt *bdiag,row,nnz,nzi,reallocs=0,nzbd,*im;
PetscReal f;
PetscInt nlnk,*lnk,k,**bi_ptr;
PetscFreeSpaceList free_space=NULL,current_space=NULL;
PetscBT lnkbt;
PetscBool missing;
PetscFunctionBegin;
if (A->rmap->N != A->cmap->N) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"matrix must be square");
ierr = MatMissingDiagonal(A,&missing,&i);CHKERRQ(ierr);
if (missing) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Matrix is missing diagonal entry %D",i);
ierr = ISInvertPermutation(iscol,PETSC_DECIDE,&isicol);CHKERRQ(ierr);
ierr = ISGetIndices(isrow,&r);CHKERRQ(ierr);
ierr = ISGetIndices(isicol,&ic);CHKERRQ(ierr);
/* get new row and diagonal pointers, must be allocated separately because they will be given to the Mat_SeqAIJ and freed separately */
ierr = PetscMalloc1(n+1,&bi);CHKERRQ(ierr);
ierr = PetscMalloc1(n+1,&bdiag);CHKERRQ(ierr);
bi[0] = bdiag[0] = 0;
/* linked list for storing column indices of the active row */
nlnk = n + 1;
ierr = PetscLLCreate(n,n,nlnk,lnk,lnkbt);CHKERRQ(ierr);
ierr = PetscMalloc2(n+1,&bi_ptr,n+1,&im);CHKERRQ(ierr);
/* initial FreeSpace size is f*(ai[n]+1) */
f = info->fill;
ierr = PetscFreeSpaceGet(PetscRealIntMultTruncate(f,ai[n]+1),&free_space);CHKERRQ(ierr);
current_space = free_space;
for (i=0; i<n; i++) {
/* copy previous fill into linked list */
nzi = 0;
nnz = ai[r[i]+1] - ai[r[i]];
ajtmp = aj + ai[r[i]];
ierr = PetscLLAddPerm(nnz,ajtmp,ic,n,nlnk,lnk,lnkbt);CHKERRQ(ierr);
nzi += nlnk;
/* add pivot rows into linked list */
row = lnk[n];
while (row < i) {
nzbd = bdiag[row] - bi[row] + 1; /* num of entries in the row with column index <= row */
ajtmp = bi_ptr[row] + nzbd; /* points to the entry next to the diagonal */
ierr = PetscLLAddSortedLU(ajtmp,row,nlnk,lnk,lnkbt,i,nzbd,im);CHKERRQ(ierr);
nzi += nlnk;
row = lnk[row];
}
bi[i+1] = bi[i] + nzi;
im[i] = nzi;
/* mark bdiag */
nzbd = 0;
nnz = nzi;
k = lnk[n];
while (nnz-- && k < i) {
nzbd++;
k = lnk[k];
}
bdiag[i] = bi[i] + nzbd;
/* if free space is not available, make more free space */
if (current_space->local_remaining<nzi) {
nnz = PetscIntMultTruncate(n - i,nzi); /* estimated and max additional space needed */
ierr = PetscFreeSpaceGet(nnz,¤t_space);CHKERRQ(ierr);
reallocs++;
}
/* copy data into free space, then initialize lnk */
ierr = PetscLLClean(n,n,nzi,lnk,current_space->array,lnkbt);CHKERRQ(ierr);
bi_ptr[i] = current_space->array;
current_space->array += nzi;
current_space->local_used += nzi;
current_space->local_remaining -= nzi;
}
#if defined(PETSC_USE_INFO)
if (ai[n] != 0) {
PetscReal af = ((PetscReal)bi[n])/((PetscReal)ai[n]);
ierr = PetscInfo3(A,"Reallocs %D Fill ratio:given %g needed %g\n",reallocs,(double)f,(double)af);CHKERRQ(ierr);
ierr = PetscInfo1(A,"Run with -pc_factor_fill %g or use \n",(double)af);CHKERRQ(ierr);
ierr = PetscInfo1(A,"PCFactorSetFill(pc,%g);\n",(double)af);CHKERRQ(ierr);
ierr = PetscInfo(A,"for best performance.\n");CHKERRQ(ierr);
} else {
ierr = PetscInfo(A,"Empty matrix\n");CHKERRQ(ierr);
}
#endif
ierr = ISRestoreIndices(isrow,&r);CHKERRQ(ierr);
ierr = ISRestoreIndices(isicol,&ic);CHKERRQ(ierr);
/* destroy list of free space and other temporary array(s) */
ierr = PetscMalloc1(bi[n]+1,&bj);CHKERRQ(ierr);
ierr = PetscFreeSpaceContiguous(&free_space,bj);CHKERRQ(ierr);
ierr = PetscLLDestroy(lnk,lnkbt);CHKERRQ(ierr);
ierr = PetscFree2(bi_ptr,im);CHKERRQ(ierr);
/* put together the new matrix */
ierr = MatSeqBAIJSetPreallocation(B,bs,MAT_SKIP_ALLOCATION,NULL);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)B,(PetscObject)isicol);CHKERRQ(ierr);
b = (Mat_SeqBAIJ*)(B)->data;
b->free_a = PETSC_TRUE;
b->free_ij = PETSC_TRUE;
b->singlemalloc = PETSC_FALSE;
ierr = PetscMalloc1((bi[n]+1)*bs2,&b->a);CHKERRQ(ierr);
b->j = bj;
b->i = bi;
b->diag = bdiag;
b->free_diag = PETSC_TRUE;
b->ilen = 0;
b->imax = 0;
b->row = isrow;
b->col = iscol;
b->pivotinblocks = (info->pivotinblocks) ? PETSC_TRUE : PETSC_FALSE;
ierr = PetscObjectReference((PetscObject)isrow);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)iscol);CHKERRQ(ierr);
b->icol = isicol;
ierr = PetscMalloc1(bs*n+bs,&b->solve_work);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)B,(bi[n]-n)*(sizeof(PetscInt)+sizeof(PetscScalar)*bs2));CHKERRQ(ierr);
b->maxnz = b->nz = bi[n];
(B)->factortype = MAT_FACTOR_LU;
(B)->info.factor_mallocs = reallocs;
(B)->info.fill_ratio_given = f;
if (ai[n] != 0) {
(B)->info.fill_ratio_needed = ((PetscReal)bi[n])/((PetscReal)ai[n]);
} else {
(B)->info.fill_ratio_needed = 0.0;
}
ierr = ISIdentity(isrow,&row_identity);CHKERRQ(ierr);
ierr = ISIdentity(iscol,&col_identity);CHKERRQ(ierr);
both_identity = (PetscBool) (row_identity && col_identity);
ierr = MatSeqBAIJSetNumericFactorization_inplace(B,both_identity);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_INTERN PetscErrorCode MatConvert_SeqSBAIJ_SeqBAIJ(Mat A, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat B;
Mat_SeqSBAIJ *a = (Mat_SeqSBAIJ*)A->data;
Mat_SeqBAIJ *b;
PetscErrorCode ierr;
PetscInt *ai=a->i,*aj=a->j,m=A->rmap->N,n=A->cmap->n,i,k,*bi,*bj,*browlengths,nz,*browstart,itmp;
PetscInt bs =A->rmap->bs,bs2=bs*bs,mbs=m/bs,col,row;
MatScalar *av,*bv;
PetscFunctionBegin;
/* compute browlengths of newmat */
ierr = PetscMalloc2(mbs,&browlengths,mbs,&browstart);
CHKERRQ(ierr);
for (i=0; i<mbs; i++) browlengths[i] = 0;
for (i=0; i<mbs; i++) {
nz = ai[i+1] - ai[i];
aj++; /* skip diagonal */
for (k=1; k<nz; k++) { /* no. of lower triangular blocks */
browlengths[*aj]++;
aj++;
}
browlengths[i] += nz; /* no. of upper triangular blocks */
}
ierr = MatCreate(PetscObjectComm((PetscObject)A),&B);
CHKERRQ(ierr);
ierr = MatSetSizes(B,m,n,m,n);
CHKERRQ(ierr);
ierr = MatSetType(B,MATSEQBAIJ);
CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(B,bs,0,browlengths);
CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_ROW_ORIENTED,PETSC_TRUE);
CHKERRQ(ierr);
b = (Mat_SeqBAIJ*)(B->data);
bi = b->i;
bj = b->j;
bv = b->a;
/* set b->i */
bi[0] = 0;
for (i=0; i<mbs; i++) {
b->ilen[i] = browlengths[i];
bi[i+1] = bi[i] + browlengths[i];
browstart[i] = bi[i];
}
if (bi[mbs] != 2*a->nz - mbs) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"bi[mbs]: %D != 2*a->nz - mbs: %D\n",bi[mbs],2*a->nz - mbs);
/* set b->j and b->a */
aj = a->j;
av = a->a;
for (i=0; i<mbs; i++) {
/* diagonal block */
*(bj + browstart[i]) = *aj;
aj++;
itmp = bs2*browstart[i];
for (k=0; k<bs2; k++) {
*(bv + itmp + k) = *av;
av++;
}
browstart[i]++;
nz = ai[i+1] - ai[i] -1;
while (nz--) {
/* lower triangular blocks - transpose blocks of A */
*(bj + browstart[*aj]) = i; /* block col index */
itmp = bs2*browstart[*aj]; /* row index */
for (col=0; col<bs; col++) {
k = col;
for (row=0; row<bs; row++) {
bv[itmp + col*bs+row] = av[k];
k+=bs;
}
}
browstart[*aj]++;
/* upper triangular blocks */
*(bj + browstart[i]) = *aj;
aj++;
itmp = bs2*browstart[i];
for (k=0; k<bs2; k++) {
bv[itmp + k] = av[k];
}
av += bs2;
browstart[i]++;
}
}
ierr = PetscFree2(browlengths,browstart);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
if (reuse == MAT_INPLACE_MATRIX) {
ierr = MatHeaderReplace(A,&B);
CHKERRQ(ierr);
} else {
*newmat = B;
}
PetscFunctionReturn(0);
}
int main(int argc, char *args[])
{
PFLOTRANMesh data;
Mat Adj; /* The adjacency matrix of the mesh */
PetscInt bs = 3;
PetscScalar values[9],*cc;
PetscMPIInt size;
PetscInt i;
PetscErrorCode ierr;
PetscViewer binaryviewer;
Vec cellCenters;
PetscViewer hdf5viewer;
hid_t file_id, dataset_id, dataspace_id;
herr_t status;
PetscFunctionBegin;
ierr = PetscInitialize(&argc, &args, (char *) 0, help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
if (size > 1) SETERRQ(PETSC_ERR_SUP,"This preprocessor runs only on one process");
/* Open Glenn's file */
ierr = PetscViewerCreate(PETSC_COMM_SELF, &hdf5viewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(hdf5viewer, PETSC_VIEWER_HDF5);CHKERRQ(ierr);
ierr = PetscViewerFileSetMode(hdf5viewer, FILE_MODE_READ);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(hdf5viewer, "mesh.h5");CHKERRQ(ierr);
ierr = PetscViewerHDF5GetFileId(hdf5viewer, &file_id);CHKERRQ(ierr);
/* get number of cells and then number of edges */
dataset_id = H5Dopen(file_id, "/Cells/Natural IDs");
dataspace_id = H5Dget_space(dataset_id);
status = H5Sget_simple_extent_dims(dataspace_id, &data.numCells, NULL);if (status < 0) SETERRQ(PETSC_ERR_LIB,"Bad dimension");
status = H5Sclose(dataspace_id);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Connections/Areas");
dataspace_id = H5Dget_space(dataset_id);
status = H5Sget_simple_extent_dims(dataspace_id, &data.numFaces, NULL);if (status < 0) SETERRQ(PETSC_ERR_LIB,"Bad dimension");
status = H5Sclose(dataspace_id);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
ierr = PetscPrintf(PETSC_COMM_SELF, "Number of cells %D Number of faces %D \n",(PetscInt)data.numCells,(PetscInt)data.numFaces);CHKERRQ(ierr);
/* read face data */
ierr = PetscMalloc5(data.numFaces,double,&data.faceAreas,data.numFaces,int,&data.downCells,data.numFaces,double,&data.downX,data.numFaces,double,&data.downY,data.numFaces,double,&data.downZ);CHKERRQ(ierr);
dataset_id = H5Dopen(file_id, "/Connections/Areas");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.faceAreas);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Connections/Downwind Cell IDs");
status = H5Dread(dataset_id, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.downCells);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Connections/Downwind Distance X");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.downX);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Connections/Downwind Distance Y");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.downY);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Connections/Downwind Distance Z");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.downZ);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
ierr = PetscMalloc4(data.numFaces,int,&data.upCells,data.numFaces,double,&data.upX,data.numFaces,double,&data.upY,data.numFaces,double,&data.upZ);CHKERRQ(ierr);
dataset_id = H5Dopen(file_id, "/Connections/Upwind Cell IDs");
status = H5Dread(dataset_id, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.upCells);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Connections/Upwind Distance X");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.upX);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Connections/Upwind Distance Y");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.upY);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Connections/Upwind Distance Z");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.upZ);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
// Put face data into matrix
ierr = MatCreate(PETSC_COMM_WORLD, &Adj);CHKERRQ(ierr);
ierr = MatSetSizes(Adj, data.numCells*bs, data.numCells*bs, PETSC_DECIDE, PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetFromOptions(Adj);CHKERRQ(ierr);
ierr = MatSetType(Adj,MATSEQBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(Adj, bs, 6,PETSC_NULL);CHKERRQ(ierr);
//ierr = MatSetType(Adj,MATSEQAIJ);CHKERRQ(ierr);
//ierr = MatSeqAIJSetPreallocation(Adj, 6,PETSC_NULL);CHKERRQ(ierr);
for(i = 0; i < data.numFaces; ++i) {
values[0] = data.faceAreas[i];
values[1] = data.downCells[i];
values[2] = data.downX[i];
values[3] = data.downY[i];
values[4] = data.downZ[i];
values[5] = data.upCells[i];
values[6] = data.upX[i];
values[7] = data.upY[i];
values[8] = data.upZ[i];
ierr = MatSetValuesBlocked(Adj, 1, &data.downCells[i], 1, &data.upCells[i], values, INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValuesBlocked(Adj, 1, &data.upCells[i], 1, &data.downCells[i], values, INSERT_VALUES);CHKERRQ(ierr);
//ierr = MatSetValues(Adj, 1, &data.downCells[i], 1, &data.upCells[i], values, INSERT_VALUES);CHKERRQ(ierr);
//ierr = MatSetValues(Adj, 1, &data.upCells[i], 1, &data.downCells[i], values, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(Adj, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Adj, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscFree5(data.faceAreas, data.downCells, data.downX, data.downY, data.downZ);CHKERRQ(ierr);
ierr = PetscFree4(data.upCells, data.upX, data.upY, data.upZ);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"mesh.petsc", FILE_MODE_WRITE,&binaryviewer);CHKERRQ(ierr);
ierr = MatView(Adj, binaryviewer);CHKERRQ(ierr);
ierr = MatDestroy(Adj);CHKERRQ(ierr);
/* read cell information */
ierr = PetscMalloc5(data.numCells,int,&data.cellIds,data.numCells,double,&data.cellVols,data.numCells,double,&data.cellX,data.numCells,double,&data.cellY,data.numCells,double,&data.cellZ);CHKERRQ(ierr);
dataset_id = H5Dopen(file_id, "/Cells/Natural IDs");
status = H5Dread(dataset_id, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.cellIds);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Cells/Volumes");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.cellVols);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Cells/X-Coordinates");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.cellX);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Cells/Y-Coordinates");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.cellY);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
dataset_id = H5Dopen(file_id, "/Cells/Z-Coordinates");
status = H5Dread(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.cellZ);CHKERRQ(status);
status = H5Dclose(dataset_id);CHKERRQ(status);
ierr = PetscViewerDestroy(hdf5viewer);CHKERRQ(ierr);
/* put cell information into vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,3*data.numCells,&cellCenters);CHKERRQ(ierr);
ierr = VecSetBlockSize(cellCenters,3);CHKERRQ(ierr);
ierr = VecGetArray(cellCenters,&cc);CHKERRQ(ierr);
for (i=0; i<data.numCells; i++) {
cc[3*i] = data.cellX[i];
cc[3*i+1] = data.cellY[i];
cc[3*i+2] = data.cellZ[i];
}
ierr = VecRestoreArray(cellCenters,&cc);CHKERRQ(ierr);
ierr = VecView(cellCenters,binaryviewer);CHKERRQ(ierr);
ierr = VecGetArray(cellCenters,&cc);CHKERRQ(ierr);
for (i=0; i<data.numCells; i++) {
cc[3*i] = data.cellIds[i];
cc[3*i+1] = data.cellVols[i];
cc[3*i+2] = 0.0;
}
ierr = VecRestoreArray(cellCenters,&cc);CHKERRQ(ierr);
ierr = VecView(cellCenters,binaryviewer);CHKERRQ(ierr);
ierr = PetscFree5(data.cellIds, data.cellVols, data.cellX, data.cellY, data.cellZ);CHKERRQ(ierr);
ierr = VecDestroy(cellCenters);
ierr = PetscViewerDestroy(binaryviewer);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
PetscMPIInt size;
PetscErrorCode ierr;
Vec x,y,b,s1,s2;
Mat A; /* linear system matrix */
Mat sA,sB,sC; /* symmetric part of the matrices */
PetscInt n,mbs=16,bs=1,nz=3,prob=1,i,j,k1,k2,col[3],lf,block, row,Ii,J,n1,inc;
PetscReal norm1,norm2,rnorm,tol=PETSC_SMALL;
PetscScalar neg_one = -1.0,four=4.0,value[3];
IS perm, iscol;
PetscRandom rdm;
PetscBool doIcc=PETSC_TRUE,equal;
MatInfo minfo1,minfo2;
MatFactorInfo factinfo;
MatType type;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"This is a uniprocessor example only!");
ierr = PetscOptionsGetInt(NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-mbs",&mbs,NULL);CHKERRQ(ierr);
n = mbs*bs;
ierr = MatCreate(PETSC_COMM_SELF,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(A,MATSEQBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,nz,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_SELF,&sA);CHKERRQ(ierr);
ierr = MatSetSizes(sA,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(sA,MATSEQSBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(sA);CHKERRQ(ierr);
ierr = MatGetType(sA,&type);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)sA,MATSEQSBAIJ,&doIcc);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(sA,bs,nz,NULL);CHKERRQ(ierr);
ierr = MatSetOption(sA,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
/* Test MatGetOwnershipRange() */
ierr = MatGetOwnershipRange(A,&Ii,&J);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(sA,&i,&j);CHKERRQ(ierr);
if (i-Ii || j-J) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetOwnershipRange() in MatSBAIJ format\n");CHKERRQ(ierr);
}
/* Assemble matrix */
if (bs == 1) {
ierr = PetscOptionsGetInt(NULL,"-test_problem",&prob,NULL);CHKERRQ(ierr);
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0]=0; col[1] = n - 2; col[2] = n - 1;
value[0]= 0.1; value[1]=-1; value[2]=2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0;
col[0] = n-1; col[1] = 1; col[2] = 0;
value[0] = 0.1; value[1] = -1.0; value[2] = 2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (PetscInt) (PetscSqrtReal((PetscReal)n) + 0.001);
if (n1*n1 - n) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"sqrt(n) must be a positive interger!");
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatSetValues(A,1,&Ii,1,&Ii,&four,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&Ii,&four,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0; value[1] = 4.0; value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = bs - 1+block*bs; col[0] = bs - 2+block*bs; col[1] = bs - 1+block*bs;
value[0]=-1.0; value[1]=4.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0+block*bs; col[0] = 0+block*bs; col[1] = 1+block*bs;
value[0]=4.0; value[1] = -1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(A,1,&i,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
col[0]=i; row=i+bs;
ierr = MatSetValues(A,1,&row,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&row,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyBegin(sA,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(sA,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatGetInfo() of A and sA */
ierr = MatGetInfo(A,MAT_LOCAL,&minfo1);CHKERRQ(ierr);
ierr = MatGetInfo(sA,MAT_LOCAL,&minfo2);CHKERRQ(ierr);
/*
printf("A matrix nonzeros (BAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo1.nz_used,(int)minfo1.nz_allocated);
printf("sA matrix nonzeros(SBAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo2.nz_used,(int)minfo2.nz_allocated);
*/
i = (int) (minfo1.nz_used - minfo2.nz_used);
j = (int) (minfo1.nz_allocated - minfo2.nz_allocated);
k1 = (int) (minfo1.nz_allocated - minfo1.nz_used);
k2 = (int) (minfo2.nz_allocated - minfo2.nz_used);
if (i < 0 || j < 0 || k1 < 0 || k2 < 0) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error (compare A and sA): MatGetInfo()\n");CHKERRQ(ierr);
}
/* Test MatDuplicate() */
ierr = MatNorm(A,NORM_FROBENIUS,&norm1);CHKERRQ(ierr);
ierr = MatDuplicate(sA,MAT_COPY_VALUES,&sB);CHKERRQ(ierr);
ierr = MatEqual(sA,sB,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDuplicate()");
/* Test MatNorm() */
ierr = MatNorm(A,NORM_FROBENIUS,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_FROBENIUS,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_FROBENIUS, NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_INFINITY,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_INFINITY,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_INFINITY(), NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_1,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_1,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_INFINITY(), NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
/* Test MatGetInfo(), MatGetSize(), MatGetBlockSize() */
ierr = MatGetInfo(A,MAT_LOCAL,&minfo1);CHKERRQ(ierr);
ierr = MatGetInfo(sB,MAT_LOCAL,&minfo2);CHKERRQ(ierr);
/*
printf("matrix nonzeros (BAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo1.nz_used,(int)minfo1.nz_allocated);
printf("matrix nonzeros(SBAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo2.nz_used,(int)minfo2.nz_allocated);
*/
i = (int) (minfo1.nz_used - minfo2.nz_used);
j = (int) (minfo1.nz_allocated - minfo2.nz_allocated);
k1 = (int) (minfo1.nz_allocated - minfo1.nz_used);
k2 = (int) (minfo2.nz_allocated - minfo2.nz_used);
if (i < 0 || j < 0 || k1 < 0 || k2 < 0) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error(compare A and sB): MatGetInfo()\n");CHKERRQ(ierr);
}
ierr = MatGetSize(A,&Ii,&J);CHKERRQ(ierr);
ierr = MatGetSize(sB,&i,&j);CHKERRQ(ierr);
if (i-Ii || j-J) {
PetscPrintf(PETSC_COMM_SELF,"Error: MatGetSize()\n");CHKERRQ(ierr);
}
ierr = MatGetBlockSize(A, &Ii);CHKERRQ(ierr);
ierr = MatGetBlockSize(sB, &i);CHKERRQ(ierr);
if (i-Ii) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetBlockSize()\n");CHKERRQ(ierr);
}
ierr = PetscRandomCreate(PETSC_COMM_SELF,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(x,&s2);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
/* Test MatDiagonalScale(), MatGetDiagonal(), MatScale() */
#if !defined(PETSC_USE_COMPLEX)
/* Scaling matrix with complex numbers results non-spd matrix,
causing crash of MatForwardSolve() and MatBackwardSolve() */
ierr = MatDiagonalScale(A,x,x);CHKERRQ(ierr);
ierr = MatDiagonalScale(sB,x,x);CHKERRQ(ierr);
ierr = MatMultEqual(A,sB,10,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDiagonalScale");
ierr = MatGetDiagonal(A,s1);CHKERRQ(ierr);
ierr = MatGetDiagonal(sB,s2);CHKERRQ(ierr);
ierr = VecAXPY(s2,neg_one,s1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm1);CHKERRQ(ierr);
if (norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatGetDiagonal(), ||s1-s2||=%G\n",norm1);CHKERRQ(ierr);
}
{
PetscScalar alpha=0.1;
ierr = MatScale(A,alpha);CHKERRQ(ierr);
ierr = MatScale(sB,alpha);CHKERRQ(ierr);
}
#endif
/* Test MatGetRowMaxAbs() */
ierr = MatGetRowMaxAbs(A,s1,NULL);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(sB,s2,NULL);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatGetRowMaxAbs() \n");CHKERRQ(ierr);
}
/* Test MatMult() */
for (i=0; i<40; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(A,x,s1);CHKERRQ(ierr);
ierr = MatMult(sB,x,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMult(), norm1-norm2: %G\n",norm1);CHKERRQ(ierr);
}
}
/* MatMultAdd() */
for (i=0; i<40; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = VecSetRandom(y,rdm);CHKERRQ(ierr);
ierr = MatMultAdd(A,x,y,s1);CHKERRQ(ierr);
ierr = MatMultAdd(sB,x,y,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMultAdd(), norm1-norm2: %G\n",norm1);CHKERRQ(ierr);
}
}
/* Test MatCholeskyFactor(), MatICCFactor() with natural ordering */
ierr = MatGetOrdering(A,MATORDERINGNATURAL,&perm,&iscol);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
norm1 = tol;
inc = bs;
/* initialize factinfo */
ierr = PetscMemzero(&factinfo,sizeof(MatFactorInfo));CHKERRQ(ierr);
for (lf=-1; lf<10; lf += inc) {
if (lf==-1) { /* Cholesky factor of sB (duplicate sA) */
factinfo.fill = 5.0;
ierr = MatGetFactor(sB,MATSOLVERPETSC,MAT_FACTOR_CHOLESKY,&sC);CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(sC,sB,perm,&factinfo);CHKERRQ(ierr);
} else if (!doIcc) break;
else { /* incomplete Cholesky factor */
factinfo.fill = 5.0;
factinfo.levels = lf;
ierr = MatGetFactor(sB,MATSOLVERPETSC,MAT_FACTOR_ICC,&sC);CHKERRQ(ierr);
ierr = MatICCFactorSymbolic(sC,sB,perm,&factinfo);CHKERRQ(ierr);
}
ierr = MatCholeskyFactorNumeric(sC,sB,&factinfo);CHKERRQ(ierr);
/* MatView(sC, PETSC_VIEWER_DRAW_WORLD); */
/* test MatGetDiagonal on numeric factor */
/*
if (lf == -1) {
ierr = MatGetDiagonal(sC,s1);CHKERRQ(ierr);
printf(" in ex74.c, diag: \n");
ierr = VecView(s1,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
*/
ierr = MatMult(sB,x,b);CHKERRQ(ierr);
/* test MatForwardSolve() and MatBackwardSolve() */
if (lf == -1) {
ierr = MatForwardSolve(sC,b,s1);CHKERRQ(ierr);
ierr = MatBackwardSolve(sC,s1,s2);CHKERRQ(ierr);
ierr = VecAXPY(s2,neg_one,x);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&norm2);CHKERRQ(ierr);
if (10*norm1 < norm2) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatForwardSolve and BackwardSolve: Norm of error=%G, bs=%d\n",norm2,bs);CHKERRQ(ierr);
}
}
/* test MatSolve() */
ierr = MatSolve(sC,b,y);CHKERRQ(ierr);
ierr = MatDestroy(&sC);CHKERRQ(ierr);
/* Check the error */
ierr = VecAXPY(y,neg_one,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm2);CHKERRQ(ierr);
/* printf("lf: %d, error: %G\n", lf,norm2); */
if (10*norm1 < norm2 && lf-inc != -1) {
ierr = PetscPrintf(PETSC_COMM_SELF,"lf=%D, %D, Norm of error=%G, %G\n",lf-inc,lf,norm1,norm2);CHKERRQ(ierr);
}
norm1 = norm2;
if (norm2 < tol && lf != -1) break;
}
ierr = ISDestroy(&perm);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sB);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatDisAssemble_MPIBAIJ(Mat A)
{
Mat_MPIBAIJ *baij = (Mat_MPIBAIJ*)A->data;
Mat B = baij->B,Bnew;
Mat_SeqBAIJ *Bbaij = (Mat_SeqBAIJ*)B->data;
PetscErrorCode ierr;
PetscInt i,j,mbs=Bbaij->mbs,n = A->cmap->N,col,*garray=baij->garray;
PetscInt bs2 = baij->bs2,*nz,ec,m = A->rmap->n;
MatScalar *a = Bbaij->a;
MatScalar *atmp;
PetscFunctionBegin;
/* free stuff related to matrix-vec multiply */
ierr = VecGetSize(baij->lvec,&ec);CHKERRQ(ierr); /* needed for PetscLogObjectMemory below */
ierr = VecDestroy(&baij->lvec);CHKERRQ(ierr); baij->lvec = 0;
ierr = VecScatterDestroy(&baij->Mvctx);CHKERRQ(ierr); baij->Mvctx = 0;
if (baij->colmap) {
#if defined(PETSC_USE_CTABLE)
ierr = PetscTableDestroy(&baij->colmap);CHKERRQ(ierr);
#else
ierr = PetscFree(baij->colmap);CHKERRQ(ierr);
ierr = PetscLogObjectMemory(A,-Bbaij->nbs*sizeof(PetscInt));CHKERRQ(ierr);
#endif
}
/* make sure that B is assembled so we can access its values */
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* invent new B and copy stuff over */
ierr = PetscMalloc(mbs*sizeof(PetscInt),&nz);CHKERRQ(ierr);
for (i=0; i<mbs; i++) {
nz[i] = Bbaij->i[i+1]-Bbaij->i[i];
}
ierr = MatCreate(PetscObjectComm((PetscObject)B),&Bnew);CHKERRQ(ierr);
ierr = MatSetSizes(Bnew,m,n,m,n);CHKERRQ(ierr);
ierr = MatSetType(Bnew,((PetscObject)B)->type_name);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(Bnew,B->rmap->bs,0,nz);CHKERRQ(ierr);
((Mat_SeqBAIJ*)Bnew->data)->nonew = Bbaij->nonew; /* Inherit insertion error options. */
ierr = MatSetOption(Bnew,MAT_ROW_ORIENTED,PETSC_FALSE);CHKERRQ(ierr);
for (i=0; i<mbs; i++) {
for (j=Bbaij->i[i]; j<Bbaij->i[i+1]; j++) {
col = garray[Bbaij->j[j]];
atmp = a + j*bs2;
ierr = MatSetValuesBlocked_SeqBAIJ(Bnew,1,&i,1,&col,atmp,B->insertmode);CHKERRQ(ierr);
}
}
ierr = MatSetOption(Bnew,MAT_ROW_ORIENTED,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscFree(nz);CHKERRQ(ierr);
ierr = PetscFree(baij->garray);CHKERRQ(ierr);
ierr = PetscLogObjectMemory(A,-ec*sizeof(PetscInt));CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscLogObjectParent(A,Bnew);CHKERRQ(ierr);
baij->B = Bnew;
A->was_assembled = PETSC_FALSE;
A->assembled = PETSC_FALSE;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
Mat seqmat,mpimat;
PetscMPIInt rank;
PetscScalar value[3],*vals;
PetscInt i,col[3],n=5,bs=1;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
/* Create seqaij matrices of size (n+rank) by n */
ierr = MatCreate(PETSC_COMM_SELF,&seqmat);CHKERRQ(ierr);
ierr = MatSetSizes(seqmat,(n+rank)*bs,PETSC_DECIDE,PETSC_DECIDE,n*bs);CHKERRQ(ierr);
ierr = MatSetFromOptions(seqmat);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(seqmat,3*bs,NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(seqmat,bs,3,NULL);CHKERRQ(ierr);
if (bs == 1) {
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(seqmat,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0] = n - 2; col[1] = n - 1;
ierr = MatSetValues(seqmat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0; col[0] = 0; col[1] = 1; value[0] = 2.0; value[1] = -1.0;
ierr = MatSetValues(seqmat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
} else {
PetscInt *rows,*cols,j;
ierr = PetscMalloc3(bs*bs,&vals,bs,&rows,bs,&cols);CHKERRQ(ierr);
/* diagonal blocks */
for (i=0; i<bs*bs; i++) vals[i] = 2.0;
for (i=0; i<n*bs; i+=bs) {
for (j=0; j<bs; j++) {rows[j] = i+j; cols[j] = i+j;}
ierr = MatSetValues(seqmat,bs,rows,bs,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
/* off-diagonal blocks */
for (i=0; i<bs*bs; i++) vals[i] = -1.0;
for (i=0; i<(n-1)*bs; i+=bs) {
for (j=0; j<bs; j++) {rows[j] = i+j; cols[j] = i+bs+j;}
ierr = MatSetValues(seqmat,bs,rows,bs,cols,vals,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree3(vals,rows,cols);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(seqmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(seqmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d] seqmat:\n",rank);CHKERRQ(ierr);
ierr = MatView(seqmat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MatCreateMPIMatConcatenateSeqMat(PETSC_COMM_WORLD,seqmat,PETSC_DECIDE,MAT_INITIAL_MATRIX,&mpimat);CHKERRQ(ierr);
ierr = MatCreateMPIMatConcatenateSeqMat(PETSC_COMM_WORLD,seqmat,PETSC_DECIDE,MAT_REUSE_MATRIX,&mpimat);CHKERRQ(ierr);
ierr = MatView(mpimat,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&seqmat);CHKERRQ(ierr);
ierr = MatDestroy(&mpimat);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
///@todo I haven't looked at the parallel code yet (lack of time right now, and i want to see it work first). / Mikael
int
PetscSparseMtrx :: buildInternalStructure(EngngModel *eModel, int di, EquationID ut, const UnknownNumberingScheme &r_s, const UnknownNumberingScheme &c_s)
{
IntArray loc;
Domain *domain = eModel->giveDomain(di);
int nelem;
if ( mtrx ) {
MatDestroy(&mtrx);
}
if ( this->kspInit ) {
KSPDestroy(&ksp);
this->kspInit = false; // force ksp to be initialized
}
this->ut = ut;
this->emodel = eModel;
this->di = di;
#ifdef __PARALLEL_MODE
if ( eModel->isParallel() ) {
OOFEM_ERROR("PetscSparseMtrx :: buildInternalStructure - Not implemented");
}
#endif
// This should be based on the numberingscheme. Also, geqs seems redundant.
// This could simplify things.
int npeqs = eModel->giveNumberOfPrescribedEquations(ut);
int neqs = eModel->giveNumberOfEquations(ut);
nRows = r_s.isDefault() ? neqs : npeqs;
nColumns = c_s.isDefault() ? neqs : npeqs;
int totalEquations = eModel->giveNumberOfEquations(ut) + eModel->giveNumberOfPrescribedEquations(ut);
//determine nonzero structure of matrix
int ii, jj;
Element *elem;
IntArray r_loc, c_loc;
std :: vector< std :: set< int > >rows(totalEquations);
std :: vector< std :: set< int > >rows_sym(totalEquations); // Only the symmetric part
nelem = domain->giveNumberOfElements();
for ( int n = 1; n <= nelem; n++ ) {
elem = domain->giveElement(n);
elem->giveLocationArray(r_loc, ut, r_s);
elem->giveLocationArray(c_loc, ut, c_s);
for ( int i = 1; i <= r_loc.giveSize(); i++ ) {
if ( ( ii = r_loc.at(i) ) ) {
for ( int j = 1; j <= c_loc.giveSize(); j++ ) {
jj = c_loc.at(j);
if ( jj ) {
rows [ ii - 1 ].insert(jj - 1);
if ( jj >= ii ) {
rows_sym [ ii - 1 ].insert(jj - 1);
}
}
}
}
}
}
// Structure from active boundary conditions.
AList<IntArray> r_locs, c_locs;
for ( int n = 1; n <= domain->giveNumberOfBoundaryConditions(); n++ ) {
ActiveBoundaryCondition *activebc = dynamic_cast<ActiveBoundaryCondition*>(domain->giveBc(n));
if (activebc) {
///@todo Deal with the CharType here.
activebc->giveLocationArrays(r_locs, c_locs, ut, TangentStiffnessMatrix, r_s, c_s, domain);
for (int k = 1; k < r_locs.giveSize(); k++) {
IntArray *krloc = r_locs.at(k);
IntArray *kcloc = c_locs.at(k);
for ( int i = 1; i <= krloc->giveSize(); i++ ) {
if ( ( ii = krloc->at(i) ) ) {
for ( int j = 1; j <= kcloc->giveSize(); j++ ) {
jj = kcloc->at(j);
if ( jj ) {
rows [ ii - 1 ].insert(jj - 1);
if ( jj >= ii ) {
rows_sym [ ii - 1 ].insert(jj - 1);
}
}
}
}
}
}
}
}
geqs = leqs = nRows;
IntArray d_nnz(leqs);
IntArray d_nnz_sym(leqs);
for ( int i = 0; i < leqs; i++ ) {
d_nnz(i) = rows [ i ].size();
d_nnz_sym(i) = rows_sym [ i ].size();
}
// create PETSc mat
MatCreate(PETSC_COMM_SELF, & mtrx);
MatSetSizes(mtrx, nRows, nColumns, nRows, nColumns);
//MatSetType(mtrx, MATSEQAIJ);
MatSetType(mtrx, MATSEQSBAIJ);
MatSetFromOptions(mtrx);
//The incompatible preallocations are ignored automatically.
MatSetUp(mtrx);
MatSeqAIJSetPreallocation( mtrx, 0, d_nnz.givePointer() );
MatSeqBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz.givePointer() ); ///@todo Not sure about PETSC_DECIDE here.
//MatSeqSBAIJSetPreallocation( mtrx, PETSC_DECIDE, 0, d_nnz_sym.givePointer() ); // Symmetry should practically never apply here.
MatSetOption(mtrx, MAT_ROW_ORIENTED, PETSC_FALSE); // To allow the insertion of values using MatSetValues in column major order
MatSetOption(mtrx, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
this->newValues = true;
return true;
}
