int main(int argc,char **argv)
{
PetscMPIInt rank,size;
PetscInt i,n,*indices;
const PetscInt *ii;
IS is,newis;
PetscBool flg;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/*
Test IS of size 0
*/
ierr = ISCreateGeneral(PETSC_COMM_SELF,0,&n,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = ISGetSize(is,&n);CHKERRQ(ierr);
if (n != 0) SETERRQ(PETSC_COMM_SELF,1,"ISGetSize");
ierr = ISDestroy(&is);CHKERRQ(ierr);
/*
Create large IS and test ISGetIndices()
*/
n = 10000 + rank;
ierr = PetscMalloc1(n,&indices);CHKERRQ(ierr);
for (i=0; i<n; i++) indices[i] = rank + i;
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = ISGetIndices(is,&ii);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (ii[i] != indices[i]) SETERRQ(PETSC_COMM_SELF,1,"ISGetIndices");
}
ierr = ISRestoreIndices(is,&ii);CHKERRQ(ierr);
/*
Check identity and permutation
*/
ierr = ISPermutation(is,&flg);CHKERRQ(ierr);
if (flg) SETERRQ(PETSC_COMM_SELF,1,"ISPermutation");
ierr = ISIdentity(is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISIdentity");
ierr = ISSetPermutation(is);CHKERRQ(ierr);
ierr = ISSetIdentity(is);CHKERRQ(ierr);
ierr = ISPermutation(is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISPermutation");
ierr = ISIdentity(is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISIdentity");
/*
Check equality of index sets
*/
ierr = ISEqual(is,is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISEqual");
/*
Sorting
*/
ierr = ISSort(is);CHKERRQ(ierr);
ierr = ISSorted(is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISSort");
/*
Thinks it is a different type?
*/
ierr = PetscObjectTypeCompare((PetscObject)is,ISSTRIDE,&flg);CHKERRQ(ierr);
if (flg) SETERRQ(PETSC_COMM_SELF,1,"ISStride");
ierr = PetscObjectTypeCompare((PetscObject)is,ISBLOCK,&flg);CHKERRQ(ierr);
if (flg) SETERRQ(PETSC_COMM_SELF,1,"ISBlock");
ierr = ISDestroy(&is);CHKERRQ(ierr);
/*
Inverting permutation
*/
for (i=0; i<n; i++) indices[i] = n - i - 1;
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = PetscFree(indices);CHKERRQ(ierr);
ierr = ISSetPermutation(is);CHKERRQ(ierr);
ierr = ISInvertPermutation(is,PETSC_DECIDE,&newis);CHKERRQ(ierr);
ierr = ISGetIndices(newis,&ii);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (ii[i] != n - i - 1) SETERRQ(PETSC_COMM_SELF,1,"ISInvertPermutation");
}
ierr = ISRestoreIndices(newis,&ii);CHKERRQ(ierr);
ierr = ISDestroy(&newis);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@
DMPlexGetOrdering - Calculate a reordering of the mesh
Collective on DM
Input Parameter:
+ dm - The DMPlex object
. otype - type of reordering, one of the following:
$ MATORDERINGNATURAL - Natural
$ MATORDERINGND - Nested Dissection
$ MATORDERING1WD - One-way Dissection
$ MATORDERINGRCM - Reverse Cuthill-McKee
$ MATORDERINGQMD - Quotient Minimum Degree
- label - [Optional] Label used to segregate ordering into sets, or NULL
Output Parameter:
. perm - The point permutation as an IS, perm[old point number] = new point number
Note: The label is used to group sets of points together by label value. This makes it easy to reorder a mesh which
has different types of cells, and then loop over each set of reordered cells for assembly.
Level: intermediate
.keywords: mesh
.seealso: MatGetOrdering()
@*/
PetscErrorCode DMPlexGetOrdering(DM dm, MatOrderingType otype, DMLabel label, IS *perm)
{
PetscInt numCells = 0;
PetscInt *start = NULL, *adjacency = NULL, *cperm, *clperm, *invclperm, *mask, *xls, pStart, pEnd, c, i;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidPointer(perm, 3);
ierr = DMPlexCreateNeighborCSR(dm, 0, &numCells, &start, &adjacency);CHKERRQ(ierr);
ierr = PetscMalloc3(numCells,&cperm,numCells,&mask,numCells*2,&xls);CHKERRQ(ierr);
if (numCells) {
/* Shift for Fortran numbering */
for (i = 0; i < start[numCells]; ++i) ++adjacency[i];
for (i = 0; i <= numCells; ++i) ++start[i];
ierr = SPARSEPACKgenrcm(&numCells, start, adjacency, cperm, mask, xls);CHKERRQ(ierr);
}
ierr = PetscFree(start);CHKERRQ(ierr);
ierr = PetscFree(adjacency);CHKERRQ(ierr);
/* Shift for Fortran numbering */
for (c = 0; c < numCells; ++c) --cperm[c];
/* Segregate */
if (label) {
IS valueIS;
const PetscInt *values;
PetscInt numValues, numPoints = 0;
PetscInt *sperm, *vsize, *voff, v;
ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
ierr = ISSort(valueIS);CHKERRQ(ierr);
ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
ierr = PetscCalloc3(numCells,&sperm,numValues,&vsize,numValues+1,&voff);CHKERRQ(ierr);
for (v = 0; v < numValues; ++v) {
ierr = DMLabelGetStratumSize(label, values[v], &vsize[v]);CHKERRQ(ierr);
if (v < numValues-1) voff[v+2] += vsize[v] + voff[v+1];
numPoints += vsize[v];
}
if (numPoints != numCells) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Label only covers %D cells < %D total", numPoints, numCells);
for (c = 0; c < numCells; ++c) {
const PetscInt oldc = cperm[c];
PetscInt val, vloc;
ierr = DMLabelGetValue(label, oldc, &val);CHKERRQ(ierr);
if (val == -1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %D not present in label", oldc);
ierr = PetscFindInt(val, numValues, values, &vloc);CHKERRQ(ierr);
if (vloc < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Value %D not present label", val);
sperm[voff[vloc+1]++] = oldc;
}
for (v = 0; v < numValues; ++v) {
if (voff[v+1] - voff[v] != vsize[v]) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of %D values found is %D != %D", values[v], voff[v+1] - voff[v], vsize[v]);
}
ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
ierr = PetscMemcpy(cperm, sperm, numCells * sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscFree3(sperm, vsize, voff);CHKERRQ(ierr);
}
/* Construct closure */
ierr = DMPlexCreateOrderingClosure_Static(dm, numCells, cperm, &clperm, &invclperm);CHKERRQ(ierr);
ierr = PetscFree3(cperm,mask,xls);CHKERRQ(ierr);
ierr = PetscFree(clperm);CHKERRQ(ierr);
/* Invert permutation */
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = ISCreateGeneral(PetscObjectComm((PetscObject) dm), pEnd-pStart, invclperm, PETSC_OWN_POINTER, perm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,Atrans,sA,*submatA,*submatsA;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt bs=1,mbs=10,ov=1,i,j,k,*rows,*cols,nd=2,*idx,rstart,rend,sz,M,N,Mbs;
PetscScalar *vals,rval,one=1.0;
IS *is1,*is2;
PetscRandom rand;
PetscBool flg,TestOverlap,TestSubMat,TestAllcols,test_sorted=PETSC_FALSE;
PetscInt vid = -1;
#if defined(PETSC_USE_LOG)
PetscLogStage stages[2];
#endif
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_mbs",&mbs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-view_id",&vid,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_overlap", &TestOverlap);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_submat", &TestSubMat);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL, "-test_allcols", &TestAllcols);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-test_sorted",&test_sorted,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,mbs*bs,mbs*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(A,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (j=0; j<bs*bs; j++) vals[j] = 0.0;
for (i=0; i<Mbs; i++) {
cols[0] = i*bs; rows[0] = i*bs;
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
/* second, add random blocks */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
cols[0] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(PetscInt)(PetscRealPart(rval)*mbs);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* make A a symmetric matrix: A <- A^T + A */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatEqual(A, Atrans, &flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric");
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* create a SeqSBAIJ matrix sA (= A) */
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
if (vid >= 0 && vid < size) {
if (!rank) printf("A: \n");
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (!rank) printf("sA: \n");
ierr = MatView(sA,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Test sA==A through MatMult() */
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Error in MatConvert(): A != sA");
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
for (i=0; i<nd; i++) {
if (!TestAllcols) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
sz = (PetscInt)((0.5+0.2*PetscRealPart(rval))*mbs); /* 0.5*mbs < sz < 0.7*mbs */
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(PetscInt)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
if (rank == vid) {
ierr = PetscPrintf(PETSC_COMM_SELF," [%d] IS sz[%d]: %d\n",rank,i,sz);CHKERRQ(ierr);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
} else { /* Test all rows and colums */
sz = M;
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is1+i);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,sz,0,1,is2+i);CHKERRQ(ierr);
if (rank == vid) {
PetscBool colflag;
ierr = ISIdentity(is2[i],&colflag);CHKERRQ(ierr);
printf("[%d] is2[%d], colflag %d\n",rank,(int)i,(int)colflag);
ierr = ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
}
}
ierr = PetscLogStageRegister("MatOv_SBAIJ",&stages[0]);CHKERRQ(ierr);
ierr = PetscLogStageRegister("MatOv_BAIJ",&stages[1]);CHKERRQ(ierr);
/* Test MatIncreaseOverlap */
if (TestOverlap) {
ierr = PetscLogStagePush(stages[0]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
ierr = PetscLogStagePush(stages[1]);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = PetscLogStagePop();CHKERRQ(ierr);
if (rank == vid) {
printf("\n[%d] IS from BAIJ:\n",rank);
ierr = ISView(is1[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
printf("\n[%d] IS from SBAIJ:\n",rank);
ierr = ISView(is2[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
if (!rank) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
/* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
ierr = ISSort(is2[i]);CHKERRQ(ierr);
/* ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
}
SETERRQ1(PETSC_COMM_SELF,1,"i=%D, is1 != is2",i);
}
}
}
/* Test MatCreateSubmatrices */
if (TestSubMat) {
if (test_sorted) {
for (i = 0; i < nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
}
}
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"A != sA");
/* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(sA,nd,is1,is1,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"MatCreateSubmatrices(): A != sA");
ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr);
ierr = MatDestroySubMatrices(nd,&submatsA);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode SNESSolve_VINEWTONRSLS(SNES snes)
{
SNES_VINEWTONRSLS *vi = (SNES_VINEWTONRSLS*)snes->data;
PetscErrorCode ierr;
PetscInt maxits,i,lits;
PetscBool lssucceed;
MatStructure flg = DIFFERENT_NONZERO_PATTERN;
PetscReal fnorm,gnorm,xnorm=0,ynorm;
Vec Y,X,F;
KSPConvergedReason kspreason;
PetscFunctionBegin;
snes->numFailures = 0;
snes->numLinearSolveFailures = 0;
snes->reason = SNES_CONVERGED_ITERATING;
maxits = snes->max_its; /* maximum number of iterations */
X = snes->vec_sol; /* solution vector */
F = snes->vec_func; /* residual vector */
Y = snes->work[0]; /* work vectors */
ierr = SNESLineSearchSetVIFunctions(snes->linesearch, SNESVIProjectOntoBounds, SNESVIComputeInactiveSetFnorm);CHKERRQ(ierr);
ierr = SNESLineSearchSetVecs(snes->linesearch, X, PETSC_NULL, PETSC_NULL, PETSC_NULL, PETSC_NULL);CHKERRQ(ierr);
ierr = SNESLineSearchSetUp(snes->linesearch);CHKERRQ(ierr);
ierr = PetscObjectTakeAccess(snes);CHKERRQ(ierr);
snes->iter = 0;
snes->norm = 0.0;
ierr = PetscObjectGrantAccess(snes);CHKERRQ(ierr);
ierr = SNESVIProjectOntoBounds(snes,X);CHKERRQ(ierr);
ierr = SNESComputeFunction(snes,X,F);CHKERRQ(ierr);
if (snes->domainerror) {
snes->reason = SNES_DIVERGED_FUNCTION_DOMAIN;
PetscFunctionReturn(0);
}
ierr = SNESVIComputeInactiveSetFnorm(snes,F,X,&fnorm);CHKERRQ(ierr);
ierr = VecNormBegin(X,NORM_2,&xnorm);CHKERRQ(ierr); /* xnorm <- ||x|| */
ierr = VecNormEnd(X,NORM_2,&xnorm);CHKERRQ(ierr);
if (PetscIsInfOrNanReal(fnorm)) SETERRQ(((PetscObject)X)->comm,PETSC_ERR_FP,"User provided compute function generated a Not-a-Number");
ierr = PetscObjectTakeAccess(snes);CHKERRQ(ierr);
snes->norm = fnorm;
ierr = PetscObjectGrantAccess(snes);CHKERRQ(ierr);
SNESLogConvHistory(snes,fnorm,0);
ierr = SNESMonitor(snes,0,fnorm);CHKERRQ(ierr);
/* set parameter for default relative tolerance convergence test */
snes->ttol = fnorm*snes->rtol;
/* test convergence */
ierr = (*snes->ops->converged)(snes,0,0.0,0.0,fnorm,&snes->reason,snes->cnvP);CHKERRQ(ierr);
if (snes->reason) PetscFunctionReturn(0);
for (i=0; i<maxits; i++) {
IS IS_act,IS_inact; /* _act -> active set _inact -> inactive set */
IS IS_redact; /* redundant active set */
VecScatter scat_act,scat_inact;
PetscInt nis_act,nis_inact;
Vec Y_act,Y_inact,F_inact;
Mat jac_inact_inact,prejac_inact_inact;
PetscBool isequal;
/* Call general purpose update function */
if (snes->ops->update) {
ierr = (*snes->ops->update)(snes, snes->iter);CHKERRQ(ierr);
}
ierr = SNESComputeJacobian(snes,X,&snes->jacobian,&snes->jacobian_pre,&flg);CHKERRQ(ierr);
/* Create active and inactive index sets */
/*original
ierr = SNESVICreateIndexSets_RS(snes,X,F,&IS_act,&IS_inact);CHKERRQ(ierr);
*/
ierr = SNESVIGetActiveSetIS(snes,X,F,&IS_act);CHKERRQ(ierr);
if (vi->checkredundancy) {
(*vi->checkredundancy)(snes,IS_act,&IS_redact,vi->ctxP);CHKERRQ(ierr);
if (IS_redact){
ierr = ISSort(IS_redact);CHKERRQ(ierr);
ierr = ISComplement(IS_redact,X->map->rstart,X->map->rend,&IS_inact);CHKERRQ(ierr);
ierr = ISDestroy(&IS_redact);CHKERRQ(ierr);
}
else {
ierr = ISComplement(IS_act,X->map->rstart,X->map->rend,&IS_inact);CHKERRQ(ierr);
}
} else {
ierr = ISComplement(IS_act,X->map->rstart,X->map->rend,&IS_inact);CHKERRQ(ierr);
}
/* Create inactive set submatrix */
ierr = MatGetSubMatrix(snes->jacobian,IS_inact,IS_inact,MAT_INITIAL_MATRIX,&jac_inact_inact);CHKERRQ(ierr);
if (0) { /* Dead code (temporary developer hack) */
IS keptrows;
ierr = MatFindNonzeroRows(jac_inact_inact,&keptrows);CHKERRQ(ierr);
if (keptrows) {
PetscInt cnt,*nrows,k;
const PetscInt *krows,*inact;
PetscInt rstart=jac_inact_inact->rmap->rstart;
ierr = MatDestroy(&jac_inact_inact);CHKERRQ(ierr);
ierr = ISDestroy(&IS_act);CHKERRQ(ierr);
ierr = ISGetLocalSize(keptrows,&cnt);CHKERRQ(ierr);
ierr = ISGetIndices(keptrows,&krows);CHKERRQ(ierr);
ierr = ISGetIndices(IS_inact,&inact);CHKERRQ(ierr);
ierr = PetscMalloc(cnt*sizeof(PetscInt),&nrows);CHKERRQ(ierr);
for (k=0; k<cnt; k++) {
nrows[k] = inact[krows[k]-rstart];
}
ierr = ISRestoreIndices(keptrows,&krows);CHKERRQ(ierr);
ierr = ISRestoreIndices(IS_inact,&inact);CHKERRQ(ierr);
ierr = ISDestroy(&keptrows);CHKERRQ(ierr);
ierr = ISDestroy(&IS_inact);CHKERRQ(ierr);
ierr = ISCreateGeneral(((PetscObject)snes)->comm,cnt,nrows,PETSC_OWN_POINTER,&IS_inact);CHKERRQ(ierr);
ierr = ISComplement(IS_inact,F->map->rstart,F->map->rend,&IS_act);CHKERRQ(ierr);
ierr = MatGetSubMatrix(snes->jacobian,IS_inact,IS_inact,MAT_INITIAL_MATRIX,&jac_inact_inact);CHKERRQ(ierr);
}
}
ierr = DMSetVI(snes->dm,IS_inact);CHKERRQ(ierr);
/* remove later */
/*
ierr = VecView(vi->xu,PETSC_VIEWER_BINARY_(((PetscObject)(vi->xu))->comm));CHKERRQ(ierr);
ierr = VecView(vi->xl,PETSC_VIEWER_BINARY_(((PetscObject)(vi->xl))->comm));CHKERRQ(ierr);
ierr = VecView(X,PETSC_VIEWER_BINARY_(((PetscObject)X)->comm));CHKERRQ(ierr);
ierr = VecView(F,PETSC_VIEWER_BINARY_(((PetscObject)F)->comm));CHKERRQ(ierr);
ierr = ISView(IS_inact,PETSC_VIEWER_BINARY_(((PetscObject)IS_inact)->comm));CHKERRQ(ierr);
*/
/* Get sizes of active and inactive sets */
ierr = ISGetLocalSize(IS_act,&nis_act);CHKERRQ(ierr);
ierr = ISGetLocalSize(IS_inact,&nis_inact);CHKERRQ(ierr);
/* Create active and inactive set vectors */
ierr = SNESCreateSubVectors_VINEWTONRSLS(snes,nis_inact,&F_inact);CHKERRQ(ierr);
ierr = SNESCreateSubVectors_VINEWTONRSLS(snes,nis_act,&Y_act);CHKERRQ(ierr);
ierr = SNESCreateSubVectors_VINEWTONRSLS(snes,nis_inact,&Y_inact);CHKERRQ(ierr);
/* Create scatter contexts */
ierr = VecScatterCreate(Y,IS_act,Y_act,PETSC_NULL,&scat_act);CHKERRQ(ierr);
ierr = VecScatterCreate(Y,IS_inact,Y_inact,PETSC_NULL,&scat_inact);CHKERRQ(ierr);
/* Do a vec scatter to active and inactive set vectors */
ierr = VecScatterBegin(scat_inact,F,F_inact,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scat_inact,F,F_inact,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterBegin(scat_act,Y,Y_act,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scat_act,Y,Y_act,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterBegin(scat_inact,Y,Y_inact,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scat_inact,Y,Y_inact,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Active set direction = 0 */
ierr = VecSet(Y_act,0);CHKERRQ(ierr);
if (snes->jacobian != snes->jacobian_pre) {
ierr = MatGetSubMatrix(snes->jacobian_pre,IS_inact,IS_inact,MAT_INITIAL_MATRIX,&prejac_inact_inact);CHKERRQ(ierr);
} else prejac_inact_inact = jac_inact_inact;
ierr = ISEqual(vi->IS_inact_prev,IS_inact,&isequal);CHKERRQ(ierr);
if (!isequal) {
ierr = SNESVIResetPCandKSP(snes,jac_inact_inact,prejac_inact_inact);CHKERRQ(ierr);
flg = DIFFERENT_NONZERO_PATTERN;
}
/* ierr = ISView(IS_inact,0);CHKERRQ(ierr); */
/* ierr = ISView(IS_act,0);CHKERRQ(ierr);*/
/* ierr = MatView(snes->jacobian_pre,0); */
ierr = KSPSetOperators(snes->ksp,jac_inact_inact,prejac_inact_inact,flg);CHKERRQ(ierr);
ierr = KSPSetUp(snes->ksp);CHKERRQ(ierr);
{
PC pc;
PetscBool flg;
ierr = KSPGetPC(snes->ksp,&pc);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)pc,PCFIELDSPLIT,&flg);CHKERRQ(ierr);
if (flg) {
KSP *subksps;
ierr = PCFieldSplitGetSubKSP(pc,PETSC_NULL,&subksps);CHKERRQ(ierr);
ierr = KSPGetPC(subksps[0],&pc);CHKERRQ(ierr);
ierr = PetscFree(subksps);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)pc,PCBJACOBI,&flg);CHKERRQ(ierr);
if (flg) {
PetscInt n,N = 101*101,j,cnts[3] = {0,0,0};
const PetscInt *ii;
ierr = ISGetSize(IS_inact,&n);CHKERRQ(ierr);
ierr = ISGetIndices(IS_inact,&ii);CHKERRQ(ierr);
for (j=0; j<n; j++) {
if (ii[j] < N) cnts[0]++;
else if (ii[j] < 2*N) cnts[1]++;
else if (ii[j] < 3*N) cnts[2]++;
}
ierr = ISRestoreIndices(IS_inact,&ii);CHKERRQ(ierr);
ierr = PCBJacobiSetTotalBlocks(pc,3,cnts);CHKERRQ(ierr);
}
}
}
ierr = SNES_KSPSolve(snes,snes->ksp,F_inact,Y_inact);CHKERRQ(ierr);
ierr = KSPGetConvergedReason(snes->ksp,&kspreason);CHKERRQ(ierr);
if (kspreason < 0) {
if (++snes->numLinearSolveFailures >= snes->maxLinearSolveFailures) {
ierr = PetscInfo2(snes,"iter=%D, number linear solve failures %D greater than current SNES allowed, stopping solve\n",snes->iter,snes->numLinearSolveFailures);CHKERRQ(ierr);
snes->reason = SNES_DIVERGED_LINEAR_SOLVE;
break;
}
}
ierr = VecScatterBegin(scat_act,Y_act,Y,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(scat_act,Y_act,Y,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(scat_inact,Y_inact,Y,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(scat_inact,Y_inact,Y,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecDestroy(&F_inact);CHKERRQ(ierr);
ierr = VecDestroy(&Y_act);CHKERRQ(ierr);
ierr = VecDestroy(&Y_inact);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scat_act);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scat_inact);CHKERRQ(ierr);
ierr = ISDestroy(&IS_act);CHKERRQ(ierr);
if (!isequal) {
ierr = ISDestroy(&vi->IS_inact_prev);CHKERRQ(ierr);
ierr = ISDuplicate(IS_inact,&vi->IS_inact_prev);CHKERRQ(ierr);
}
ierr = ISDestroy(&IS_inact);CHKERRQ(ierr);
ierr = MatDestroy(&jac_inact_inact);CHKERRQ(ierr);
if (snes->jacobian != snes->jacobian_pre) {
ierr = MatDestroy(&prejac_inact_inact);CHKERRQ(ierr);
}
ierr = KSPGetIterationNumber(snes->ksp,&lits);CHKERRQ(ierr);
snes->linear_its += lits;
ierr = PetscInfo2(snes,"iter=%D, linear solve iterations=%D\n",snes->iter,lits);CHKERRQ(ierr);
/*
if (snes->ops->precheck) {
PetscBool changed_y = PETSC_FALSE;
ierr = (*snes->ops->precheck)(snes,X,Y,snes->precheck,&changed_y);CHKERRQ(ierr);
}
if (PetscLogPrintInfo){
ierr = SNESVICheckResidual_Private(snes,snes->jacobian,F,Y,G,W);CHKERRQ(ierr);
}
*/
/* Compute a (scaled) negative update in the line search routine:
Y <- X - lambda*Y
and evaluate G = function(Y) (depends on the line search).
*/
ierr = VecCopy(Y,snes->vec_sol_update);CHKERRQ(ierr);
ynorm = 1; gnorm = fnorm;
ierr = SNESLineSearchApply(snes->linesearch, X, F, &gnorm, Y);CHKERRQ(ierr);
ierr = SNESLineSearchGetNorms(snes->linesearch, &xnorm, &gnorm, &ynorm);CHKERRQ(ierr);
ierr = PetscInfo4(snes,"fnorm=%18.16e, gnorm=%18.16e, ynorm=%18.16e, lssucceed=%d\n",(double)fnorm,(double)gnorm,(double)ynorm,(int)lssucceed);CHKERRQ(ierr);
if (snes->reason == SNES_DIVERGED_FUNCTION_COUNT) break;
if (snes->domainerror) {
snes->reason = SNES_DIVERGED_FUNCTION_DOMAIN;
ierr = DMDestroyVI(snes->dm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
ierr = SNESLineSearchGetSuccess(snes->linesearch, &lssucceed);CHKERRQ(ierr);
if (!lssucceed) {
if (++snes->numFailures >= snes->maxFailures) {
PetscBool ismin;
snes->reason = SNES_DIVERGED_LINE_SEARCH;
ierr = SNESVICheckLocalMin_Private(snes,snes->jacobian,F,X,gnorm,&ismin);CHKERRQ(ierr);
if (ismin) snes->reason = SNES_DIVERGED_LOCAL_MIN;
break;
}
}
/* Update function and solution vectors */
fnorm = gnorm;
/* Monitor convergence */
ierr = PetscObjectTakeAccess(snes);CHKERRQ(ierr);
snes->iter = i+1;
snes->norm = fnorm;
ierr = PetscObjectGrantAccess(snes);CHKERRQ(ierr);
SNESLogConvHistory(snes,snes->norm,lits);
ierr = SNESMonitor(snes,snes->iter,snes->norm);CHKERRQ(ierr);
/* Test for convergence, xnorm = || X || */
if (snes->ops->converged != SNESSkipConverged) { ierr = VecNorm(X,NORM_2,&xnorm);CHKERRQ(ierr); }
ierr = (*snes->ops->converged)(snes,snes->iter,xnorm,ynorm,fnorm,&snes->reason,snes->cnvP);CHKERRQ(ierr);
if (snes->reason) break;
}
ierr = DMDestroyVI(snes->dm);CHKERRQ(ierr);
if (i == maxits) {
ierr = PetscInfo1(snes,"Maximum number of iterations has been reached: %D\n",maxits);CHKERRQ(ierr);
if (!snes->reason) snes->reason = SNES_DIVERGED_MAX_IT;
}
PetscFunctionReturn(0);
}
/*
DMDAGetFaceInterpolation - Gets the interpolation for a face based coarse space
*/
PetscErrorCode DMDAGetFaceInterpolation(DM da,PC_Exotic *exotic,Mat Aglobal,MatReuse reuse,Mat *P)
{
PetscErrorCode ierr;
PetscInt dim,i,j,k,m,n,p,dof,Nint,Nface,Nwire,Nsurf,*Iint,*Isurf,cint = 0,csurf = 0,istart,jstart,kstart,*II,N,c = 0;
PetscInt mwidth,nwidth,pwidth,cnt,mp,np,pp,Ntotal,gl[6],*globals,Ng,*IIint,*IIsurf,Nt;
Mat Xint, Xsurf,Xint_tmp;
IS isint,issurf,is,row,col;
ISLocalToGlobalMapping ltg;
MPI_Comm comm;
Mat A,Aii,Ais,Asi,*Aholder,iAii;
MatFactorInfo info;
PetscScalar *xsurf,*xint;
#if defined(PETSC_USE_DEBUG_foo)
PetscScalar tmp;
#endif
PetscTable ht;
PetscFunctionBegin;
ierr = DMDAGetInfo(da,&dim,0,0,0,&mp,&np,&pp,&dof,0,0,0,0,0);
CHKERRQ(ierr);
if (dof != 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only for single field problems");
if (dim != 3) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only coded for 3d problems");
ierr = DMDAGetCorners(da,0,0,0,&m,&n,&p);
CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&istart,&jstart,&kstart,&mwidth,&nwidth,&pwidth);
CHKERRQ(ierr);
istart = istart ? -1 : 0;
jstart = jstart ? -1 : 0;
kstart = kstart ? -1 : 0;
/*
the columns of P are the interpolation of each coarse grid point (one for each vertex and edge)
to all the local degrees of freedom (this includes the vertices, edges and faces).
Xint are the subset of the interpolation into the interior
Xface are the interpolation onto faces but not into the interior
Xsurf are the interpolation onto the vertices and edges (the surfbasket)
Xint
Symbolically one could write P = (Xface) after interchanging the rows to match the natural ordering on the domain
Xsurf
*/
N = (m - istart)*(n - jstart)*(p - kstart);
Nint = (m-2-istart)*(n-2-jstart)*(p-2-kstart);
Nface = 2*((m-2-istart)*(n-2-jstart) + (m-2-istart)*(p-2-kstart) + (n-2-jstart)*(p-2-kstart));
Nwire = 4*((m-2-istart) + (n-2-jstart) + (p-2-kstart)) + 8;
Nsurf = Nface + Nwire;
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nint,6,NULL,&Xint);
CHKERRQ(ierr);
ierr = MatCreateSeqDense(MPI_COMM_SELF,Nsurf,6,NULL,&Xsurf);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/*
Require that all 12 edges and 6 faces have at least one grid point. Otherwise some of the columns of
Xsurf will be all zero (thus making the coarse matrix singular).
*/
if (m-istart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in X direction must be at least 3");
if (n-jstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Y direction must be at least 3");
if (p-kstart < 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Number of grid points per process in Z direction must be at least 3");
cnt = 0;
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 0*Nsurf] = 1;
}
for (k=1; k<p-1-kstart; k++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 1*Nsurf] = 1;
for (j=1; j<n-1-jstart; j++) {
xsurf[cnt++ + 2*Nsurf] = 1;
/* these are the interior nodes */
xsurf[cnt++ + 3*Nsurf] = 1;
}
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 4*Nsurf] = 1;
}
for (j=1; j<n-1-jstart; j++) {
for (i=1; i<m-istart-1; i++) xsurf[cnt++ + 5*Nsurf] = 1;
}
#if defined(PETSC_USE_DEBUG_foo)
for (i=0; i<Nsurf; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xsurf[i+j*Nsurf];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xsurf interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
#endif
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
/* ierr = MatView(Xsurf,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
/*
I are the indices for all the needed vertices (in global numbering)
Iint are the indices for the interior values, I surf for the surface values
(This is just for the part of the global matrix obtained with MatGetSubMatrix(), it
is NOT the local DMDA ordering.)
IIint and IIsurf are the same as the Iint, Isurf except they are in the global numbering
*/
#define Endpoint(a,start,b) (a == 0 || a == (b-1-start))
ierr = PetscMalloc3(N,&II,Nint,&Iint,Nsurf,&Isurf);
CHKERRQ(ierr);
ierr = PetscMalloc2(Nint,&IIint,Nsurf,&IIsurf);
CHKERRQ(ierr);
for (k=0; k<p-kstart; k++) {
for (j=0; j<n-jstart; j++) {
for (i=0; i<m-istart; i++) {
II[c++] = i + j*mwidth + k*mwidth*nwidth;
if (!Endpoint(i,istart,m) && !Endpoint(j,jstart,n) && !Endpoint(k,kstart,p)) {
IIint[cint] = i + j*mwidth + k*mwidth*nwidth;
Iint[cint++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
} else {
IIsurf[csurf] = i + j*mwidth + k*mwidth*nwidth;
Isurf[csurf++] = i + j*(m-istart) + k*(m-istart)*(n-jstart);
}
}
}
}
if (c != N) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"c != N");
if (cint != Nint) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"cint != Nint");
if (csurf != Nsurf) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"csurf != Nsurf");
ierr = DMGetLocalToGlobalMapping(da,<g);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,N,II,II);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nint,IIint,IIint);
CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(ltg,Nsurf,IIsurf,IIsurf);
CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)da,&comm);
CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,N,II,PETSC_COPY_VALUES,&is);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nint,Iint,PETSC_COPY_VALUES,&isint);
CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,Nsurf,Isurf,PETSC_COPY_VALUES,&issurf);
CHKERRQ(ierr);
ierr = PetscFree3(II,Iint,Isurf);
CHKERRQ(ierr);
ierr = ISSort(is);
CHKERRQ(ierr);
ierr = MatGetSubMatrices(Aglobal,1,&is,&is,MAT_INITIAL_MATRIX,&Aholder);
CHKERRQ(ierr);
A = *Aholder;
ierr = PetscFree(Aholder);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,isint,MAT_INITIAL_MATRIX,&Aii);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isint,issurf,MAT_INITIAL_MATRIX,&Ais);
CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,issurf,isint,MAT_INITIAL_MATRIX,&Asi);
CHKERRQ(ierr);
/*
Solve for the interpolation onto the interior Xint
*/
ierr = MatMatMult(Ais,Xsurf,MAT_INITIAL_MATRIX,PETSC_DETERMINE,&Xint_tmp);
CHKERRQ(ierr);
ierr = MatScale(Xint_tmp,-1.0);
CHKERRQ(ierr);
if (exotic->directSolve) {
ierr = MatGetFactor(Aii,MATSOLVERPETSC,MAT_FACTOR_LU,&iAii);
CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&info);
CHKERRQ(ierr);
ierr = MatGetOrdering(Aii,MATORDERINGND,&row,&col);
CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(iAii,Aii,row,col,&info);
CHKERRQ(ierr);
ierr = ISDestroy(&row);
CHKERRQ(ierr);
ierr = ISDestroy(&col);
CHKERRQ(ierr);
ierr = MatLUFactorNumeric(iAii,Aii,&info);
CHKERRQ(ierr);
ierr = MatMatSolve(iAii,Xint_tmp,Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&iAii);
CHKERRQ(ierr);
} else {
Vec b,x;
PetscScalar *xint_tmp;
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&x);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,Nint,0,&b);
CHKERRQ(ierr);
ierr = KSPSetOperators(exotic->ksp,Aii,Aii);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = VecPlaceArray(x,xint+i*Nint);
CHKERRQ(ierr);
ierr = VecPlaceArray(b,xint_tmp+i*Nint);
CHKERRQ(ierr);
ierr = KSPSolve(exotic->ksp,b,x);
CHKERRQ(ierr);
ierr = VecResetArray(x);
CHKERRQ(ierr);
ierr = VecResetArray(b);
CHKERRQ(ierr);
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint_tmp,&xint_tmp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&b);
CHKERRQ(ierr);
}
ierr = MatDestroy(&Xint_tmp);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
for (i=0; i<Nint; i++) {
tmp = 0.0;
for (j=0; j<6; j++) tmp += xint[i+j*Nint];
if (PetscAbsScalar(tmp-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong Xint interpolation at i %D value %g",i,(double)PetscAbsScalar(tmp));
}
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
/* ierr =MatView(Xint,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
#endif
/* total faces */
Ntotal = mp*np*(pp+1) + mp*pp*(np+1) + np*pp*(mp+1);
/*
For each vertex, edge, face on process (in the same orderings as used above) determine its local number including ghost points
*/
cnt = 0;
{
gl[cnt++] = mwidth+1;
}
{
{
gl[cnt++] = mwidth*nwidth+1;
}
{
gl[cnt++] = mwidth*nwidth + mwidth; /* these are the interior nodes */ gl[cnt++] = mwidth*nwidth + mwidth+m-istart-1;
}
{
gl[cnt++] = mwidth*nwidth+mwidth*(n-jstart-1)+1;
}
}
{
gl[cnt++] = mwidth*nwidth*(p-kstart-1) + mwidth+1;
}
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* convert that to global numbering and get them on all processes */
ierr = ISLocalToGlobalMappingApply(ltg,6,gl,gl);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
ierr = PetscMalloc1(6*mp*np*pp,&globals);
CHKERRQ(ierr);
ierr = MPI_Allgather(gl,6,MPIU_INT,globals,6,MPIU_INT,PetscObjectComm((PetscObject)da));
CHKERRQ(ierr);
/* Number the coarse grid points from 0 to Ntotal */
ierr = MatGetSize(Aglobal,&Nt,NULL);
CHKERRQ(ierr);
ierr = PetscTableCreate(Ntotal/3,Nt+1,&ht);
CHKERRQ(ierr);
for (i=0; i<6*mp*np*pp; i++) {
ierr = PetscTableAddCount(ht,globals[i]+1);
CHKERRQ(ierr);
}
ierr = PetscTableGetCount(ht,&cnt);
CHKERRQ(ierr);
if (cnt != Ntotal) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Hash table size %D not equal to total number coarse grid points %D",cnt,Ntotal);
ierr = PetscFree(globals);
CHKERRQ(ierr);
for (i=0; i<6; i++) {
ierr = PetscTableFind(ht,gl[i]+1,&gl[i]);
CHKERRQ(ierr);
gl[i]--;
}
ierr = PetscTableDestroy(&ht);
CHKERRQ(ierr);
/* PetscIntView(6,gl,PETSC_VIEWER_STDOUT_WORLD); */
/* construct global interpolation matrix */
ierr = MatGetLocalSize(Aglobal,&Ng,NULL);
CHKERRQ(ierr);
if (reuse == MAT_INITIAL_MATRIX) {
ierr = MatCreateAIJ(PetscObjectComm((PetscObject)da),Ng,PETSC_DECIDE,PETSC_DECIDE,Ntotal,Nint+Nsurf,NULL,Nint,NULL,P);
CHKERRQ(ierr);
} else {
ierr = MatZeroEntries(*P);
CHKERRQ(ierr);
}
ierr = MatSetOption(*P,MAT_ROW_ORIENTED,PETSC_FALSE);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nint,IIint,6,gl,xint,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xint,&xint);
CHKERRQ(ierr);
ierr = MatDenseGetArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatSetValues(*P,Nsurf,IIsurf,6,gl,xsurf,INSERT_VALUES);
CHKERRQ(ierr);
ierr = MatDenseRestoreArray(Xsurf,&xsurf);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = PetscFree2(IIint,IIsurf);
CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG_foo)
{
Vec x,y;
PetscScalar *yy;
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),Ng,PETSC_DETERMINE,&y);
CHKERRQ(ierr);
ierr = VecCreateMPI(PetscObjectComm((PetscObject)da),PETSC_DETERMINE,Ntotal,&x);
CHKERRQ(ierr);
ierr = VecSet(x,1.0);
CHKERRQ(ierr);
ierr = MatMult(*P,x,y);
CHKERRQ(ierr);
ierr = VecGetArray(y,&yy);
CHKERRQ(ierr);
for (i=0; i<Ng; i++) {
if (PetscAbsScalar(yy[i]-1.0) > 1.e-10) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong p interpolation at i %D value %g",i,(double)PetscAbsScalar(yy[i]));
}
ierr = VecRestoreArray(y,&yy);
CHKERRQ(ierr);
ierr = VecDestroy(x);
CHKERRQ(ierr);
ierr = VecDestroy(y);
CHKERRQ(ierr);
}
#endif
ierr = MatDestroy(&Aii);
CHKERRQ(ierr);
ierr = MatDestroy(&Ais);
CHKERRQ(ierr);
ierr = MatDestroy(&Asi);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = ISDestroy(&is);
CHKERRQ(ierr);
ierr = ISDestroy(&isint);
CHKERRQ(ierr);
ierr = ISDestroy(&issurf);
CHKERRQ(ierr);
ierr = MatDestroy(&Xint);
CHKERRQ(ierr);
ierr = MatDestroy(&Xsurf);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,B,*submatA,*submatB;
PetscInt bs=1,m=11,ov=1,i,j,k,*rows,*cols,nd=5,*idx,rstart,rend,sz,mm,nn,M,N,Mbs;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscScalar *vals,rval;
IS *is1,*is2;
PetscRandom rdm;
Vec xx,s1,s2;
PetscReal s1norm,s2norm,rnorm,tol = 100*PETSC_SMALL;
PetscBool flg,test_nd0=PETSC_FALSE;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_block_size",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-mat_size",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ov",&ov,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-nd",&nd,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-test_nd0",&test_nd0,NULL);CHKERRQ(ierr);
/* Create a AIJ matrix A */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m*bs,m*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
/* Create a BAIJ matrix B */
ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,m*bs,m*bs,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(B,MATBAIJ);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(B,bs,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(B,bs,PETSC_DEFAULT,NULL,PETSC_DEFAULT,NULL);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
Mbs = M/bs;
ierr = PetscMalloc1(bs,&rows);CHKERRQ(ierr);
ierr = PetscMalloc1(bs,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*bs,&vals);CHKERRQ(ierr);
ierr = PetscMalloc1(M,&idx);CHKERRQ(ierr);
/* Now set blocks of values */
for (i=0; i<40*bs; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
cols[0] = bs*(int)(PetscRealPart(rval)*Mbs);
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
rows[0] = rstart + bs*(int)(PetscRealPart(rval)*m);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(B,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc1(nd,&is1);CHKERRQ(ierr);
ierr = PetscMalloc1(nd,&is2);CHKERRQ(ierr);
if (!rank && test_nd0) nd = 0; /* test case */
for (i=0; i<nd; i++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
sz = (int)(PetscRealPart(rval)*m);
for (j=0; j<sz; j++) {
ierr = PetscRandomGetValue(rdm,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(int)(PetscRealPart(rval)*Mbs);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,sz*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
}
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(B,nd,is2,ov);CHKERRQ(ierr);
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg) {
ierr = PetscPrintf(PETSC_COMM_SELF,"i=%D, flg=%d :bs=%D m=%D ov=%D nd=%D np=%D\n",i,flg,bs,m,ov,nd,size);CHKERRQ(ierr);
}
}
for (i=0; i<nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
ierr = ISSort(is2[i]);CHKERRQ(ierr);
}
ierr = MatCreateSubMatrices(B,nd,is2,is2,MAT_INITIAL_MATRIX,&submatB);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatB[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d]Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",rank,s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Now test MatCreateSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatCreateSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatCreateSubMatrices(B,nd,is2,is2,MAT_REUSE_MATRIX,&submatB);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rdm);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatB[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d]Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",rank,s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
}
ierr = MatDestroySubMatrices(nd,&submatA);CHKERRQ(ierr);
ierr = MatDestroySubMatrices(nd,&submatB);CHKERRQ(ierr);
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat A,Atrans,sA,*submatA,*submatsA;
PetscInt bs=1,m=43,ov=1,i,j,k,*rows,*cols,M,nd=5,*idx,mm,nn;
PetscErrorCode ierr;
PetscMPIInt size;
PetscScalar *vals,rval,one=1.0;
IS *is1,*is2;
PetscRandom rand;
Vec xx,s1,s2;
PetscReal s1norm,s2norm,rnorm,tol = 1.e-10;
PetscBool flg;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_block_size",&bs,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_size",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-ov",&ov,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-nd",&nd,PETSC_NULL);CHKERRQ(ierr);
/* create a SeqBAIJ matrix A */
M = m*bs;
ierr = MatCreateSeqBAIJ(PETSC_COMM_SELF,bs,M,M,1,PETSC_NULL,&A);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_SELF,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = PetscMalloc(bs*sizeof(PetscInt),&rows);CHKERRQ(ierr);
ierr = PetscMalloc(bs*sizeof(PetscInt),&cols);CHKERRQ(ierr);
ierr = PetscMalloc(bs*bs*sizeof(PetscScalar),&vals);CHKERRQ(ierr);
ierr = PetscMalloc(M*sizeof(PetscScalar),&idx);CHKERRQ(ierr);
/* Now set blocks of random values */
/* first, set diagonal blocks as zero */
for (j=0; j<bs*bs; j++) vals[j] = 0.0;
for (i=0; i<m; i++){
cols[0] = i*bs; rows[0] = i*bs;
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
/* second, add random blocks */
for (i=0; i<20*bs; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
cols[0] = bs*(int)(PetscRealPart(rval)*m);
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
rows[0] = bs*(int)(PetscRealPart(rval)*m);
for (j=1; j<bs; j++) {
rows[j] = rows[j-1]+1;
cols[j] = cols[j-1]+1;
}
for (j=0; j<bs*bs; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
vals[j] = rval;
}
ierr = MatSetValues(A,bs,rows,bs,cols,vals,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* make A a symmetric matrix: A <- A^T + A */
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);CHKERRQ(ierr);
ierr = MatAXPY(A,one,Atrans,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Atrans);
ierr = MatEqual(A, Atrans, &flg);
if (!flg) {
SETERRQ(PETSC_COMM_SELF,1,"A+A^T is non-symmetric");
}
ierr = MatDestroy(&Atrans);CHKERRQ(ierr);
/* create a SeqSBAIJ matrix sA (= A) */
ierr = MatConvert(A,MATSEQSBAIJ,MAT_INITIAL_MATRIX,&sA);CHKERRQ(ierr);
/* Test sA==A through MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(A,&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rand);CHKERRQ(ierr);
ierr = MatMult(A,xx,s1);CHKERRQ(ierr);
ierr = MatMult(sA,xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Test MatIncreaseOverlap() */
ierr = PetscMalloc(nd*sizeof(IS **),&is1);CHKERRQ(ierr);
ierr = PetscMalloc(nd*sizeof(IS **),&is2);CHKERRQ(ierr);
for (i=0; i<nd; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
size = (int)(PetscRealPart(rval)*m);
for (j=0; j<size; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
idx[j*bs] = bs*(int)(PetscRealPart(rval)*m);
for (k=1; k<bs; k++) idx[j*bs+k] = idx[j*bs]+k;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,size*bs,idx,PETSC_COPY_VALUES,is1+i);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,size*bs,idx,PETSC_COPY_VALUES,is2+i);CHKERRQ(ierr);
}
/* for debugging */
/*
ierr = MatView(A,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatView(sA,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
*/
ierr = MatIncreaseOverlap(A,nd,is1,ov);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(sA,nd,is2,ov);CHKERRQ(ierr);
for (i=0; i<nd; ++i) {
ierr = ISSort(is1[i]);CHKERRQ(ierr);
ierr = ISSort(is2[i]);CHKERRQ(ierr);
}
for (i=0; i<nd; ++i) {
ierr = ISEqual(is1[i],is2[i],&flg);CHKERRQ(ierr);
if (!flg){
/* ISView(is1[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ISView(is2[i],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); */
SETERRQ1(PETSC_COMM_SELF,1,"i=%d, is1 != is2",i);
}
}
ierr = MatGetSubMatrices(A,nd,is1,is1,MAT_INITIAL_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatGetSubMatrices(sA,nd,is2,is2,MAT_INITIAL_MATRIX,&submatsA);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rand);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatsA[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Now test MatGetSubmatrices with MAT_REUSE_MATRIX option */
ierr = MatGetSubMatrices(A,nd,is1,is1,MAT_REUSE_MATRIX,&submatA);CHKERRQ(ierr);
ierr = MatGetSubMatrices(sA,nd,is2,is2,MAT_REUSE_MATRIX,&submatsA);CHKERRQ(ierr);
/* Test MatMult() */
for (i=0; i<nd; i++) {
ierr = MatGetSize(submatA[i],&mm,&nn);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,mm,&xx);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(xx,&s2);CHKERRQ(ierr);
for (j=0; j<3; j++) {
ierr = VecSetRandom(xx,rand);CHKERRQ(ierr);
ierr = MatMult(submatA[i],xx,s1);CHKERRQ(ierr);
ierr = MatMult(submatsA[i],xx,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_2,&s1norm);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&s2norm);CHKERRQ(ierr);
rnorm = s2norm-s1norm;
if (rnorm<-tol || rnorm>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMult - Norm1=%16.14e Norm2=%16.14e\n",s1norm,s2norm);CHKERRQ(ierr);
}
}
ierr = VecDestroy(&xx);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
}
/* Free allocated memory */
for (i=0; i<nd; ++i) {
ierr = ISDestroy(&is1[i]);CHKERRQ(ierr);
ierr = ISDestroy(&is2[i]);CHKERRQ(ierr);
ierr = MatDestroy(&submatA[i]);CHKERRQ(ierr);
ierr = MatDestroy(&submatsA[i]);CHKERRQ(ierr);
}
ierr = PetscFree(submatA);CHKERRQ(ierr);
ierr = PetscFree(submatsA);CHKERRQ(ierr);
ierr = PetscFree(is1);CHKERRQ(ierr);
ierr = PetscFree(is2);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscFree(vals);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode PCGAMGProlongator_Classical_Standard(PC pc, const Mat A, const Mat G, PetscCoarsenData *agg_lists,Mat *P)
{
PetscErrorCode ierr;
Mat *lA;
Vec lv,v,cv;
PetscScalar *lcid;
IS lis;
PetscInt fs,fe,cs,ce,nl,i,j,k,li,lni,ci;
VecScatter lscat;
PetscInt fn,cn,cid,c_indx;
PetscBool iscoarse;
PetscScalar c_scalar;
const PetscScalar *vcol;
const PetscInt *icol;
const PetscInt *gidx;
PetscInt ncols;
PetscInt *lsparse,*gsparse;
MatType mtype;
PetscInt maxcols;
PetscReal diag,jdiag,jwttotal;
PetscScalar *pvcol,vi;
PetscInt *picol;
PetscInt pncols;
PetscScalar *pcontrib,pentry,pjentry;
/* PC_MG *mg = (PC_MG*)pc->data; */
/* PC_GAMG *gamg = (PC_GAMG*)mg->innerctx; */
PetscFunctionBegin;
ierr = MatGetOwnershipRange(A,&fs,&fe);CHKERRQ(ierr);
fn = fe-fs;
ierr = MatGetVecs(A,NULL,&v);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,fe-fs,fs,1,&lis);CHKERRQ(ierr);
/* increase the overlap by two to get neighbors of neighbors */
ierr = MatIncreaseOverlap(A,1,&lis,2);CHKERRQ(ierr);
ierr = ISSort(lis);CHKERRQ(ierr);
/* get the local part of A */
ierr = MatGetSubMatrices(A,1,&lis,&lis,MAT_INITIAL_MATRIX,&lA);CHKERRQ(ierr);
/* build the scatter out of it */
ierr = ISGetLocalSize(lis,&nl);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,nl,&lv);CHKERRQ(ierr);
ierr = VecScatterCreate(v,lis,lv,NULL,&lscat);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&lsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&gsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*nl,&pcontrib);CHKERRQ(ierr);
/* create coarse vector */
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse);CHKERRQ(ierr);
if (!iscoarse) {
cn++;
}
}
ierr = VecCreateMPI(PetscObjectComm((PetscObject)A),cn,PETSC_DECIDE,&cv);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(cv,&cs,&ce);CHKERRQ(ierr);
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
if (!iscoarse) {
cid = cs+cn;
cn++;
} else {
cid = -1;
}
*(PetscInt*)&c_scalar = cid;
c_indx = fs+i;
ierr = VecSetValues(v,1,&c_indx,&c_scalar,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecScatterBegin(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* count to preallocate the prolongator */
ierr = ISGetIndices(lis,&gidx);CHKERRQ(ierr);
ierr = VecGetArray(lv,&lcid);CHKERRQ(ierr);
maxcols = 0;
/* count the number of unique contributing coarse cells for each fine */
for (i=0;i<nl;i++) {
pcontrib[i] = 0.;
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
if (gidx[i] >= fs && gidx[i] < fe) {
li = gidx[i] - fs;
lsparse[li] = 0;
gsparse[li] = 0;
cid = *(PetscInt*)&(lcid[i]);
if (cid >= 0) {
lsparse[li] = 1;
} else {
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0) {
pcontrib[icol[j]] = 1.;
} else {
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0) {
pcontrib[icol[k]] = 1.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
}
}
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0 && pcontrib[icol[j]] != 0.) {
lni = *(PetscInt*)&(lcid[icol[j]]);
if (lni >= cs && lni < ce) {
lsparse[li]++;
} else {
gsparse[li]++;
}
pcontrib[icol[j]] = 0.;
} else {
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && pcontrib[icol[k]] != 0.) {
lni = *(PetscInt*)&(lcid[icol[k]]);
if (lni >= cs && lni < ce) {
lsparse[li]++;
} else {
gsparse[li]++;
}
pcontrib[icol[k]] = 0.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
}
}
}
if (lsparse[li] + gsparse[li] > maxcols) maxcols = lsparse[li]+gsparse[li];
}
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
ierr = PetscMalloc(sizeof(PetscInt)*maxcols,&picol);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*maxcols,&pvcol);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)A),P);CHKERRQ(ierr);
ierr = MatGetType(A,&mtype);CHKERRQ(ierr);
ierr = MatSetType(*P,mtype);CHKERRQ(ierr);
ierr = MatSetSizes(*P,fn,cn,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*P,0,lsparse,0,gsparse);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*P,0,lsparse);CHKERRQ(ierr);
for (i=0;i<nl;i++) {
diag = 0.;
if (gidx[i] >= fs && gidx[i] < fe) {
li = gidx[i] - fs;
pncols=0;
cid = *(PetscInt*)&(lcid[i]);
if (cid >= 0) {
pncols = 1;
picol[0] = cid;
pvcol[0] = 1.;
} else {
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
pentry = vcol[j];
if (*(PetscInt*)&(lcid[icol[j]]) >= 0) {
/* coarse neighbor */
pcontrib[icol[j]] += pentry;
} else if (icol[j] != i) {
/* the neighbor is a strongly connected fine node */
ci = icol[j];
vi = vcol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
jwttotal=0.;
jdiag = 0.;
for (k=0;k<ncols;k++) {
if (ci == icol[k]) {
jdiag = PetscRealPart(vcol[k]);
}
}
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && jdiag*PetscRealPart(vcol[k]) < 0.) {
pjentry = vcol[k];
jwttotal += PetscRealPart(pjentry);
}
}
if (jwttotal != 0.) {
jwttotal = PetscRealPart(vi)/jwttotal;
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && jdiag*PetscRealPart(vcol[k]) < 0.) {
pjentry = vcol[k]*jwttotal;
pcontrib[icol[k]] += pjentry;
}
}
} else {
diag += PetscRealPart(vi);
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
} else {
diag += PetscRealPart(vcol[j]);
}
}
if (diag != 0.) {
diag = 1./diag;
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0 && pcontrib[icol[j]] != 0.) {
/* the neighbor is a coarse node */
if (PetscAbsScalar(pcontrib[icol[j]]) > 0.0) {
lni = *(PetscInt*)&(lcid[icol[j]]);
pvcol[pncols] = -pcontrib[icol[j]]*diag;
picol[pncols] = lni;
pncols++;
}
pcontrib[icol[j]] = 0.;
} else {
/* the neighbor is a strongly connected fine node */
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && pcontrib[icol[k]] != 0.) {
if (PetscAbsScalar(pcontrib[icol[k]]) > 0.0) {
lni = *(PetscInt*)&(lcid[icol[k]]);
pvcol[pncols] = -pcontrib[icol[k]]*diag;
picol[pncols] = lni;
pncols++;
}
pcontrib[icol[k]] = 0.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
pcontrib[icol[j]] = 0.;
}
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
}
ci = gidx[i];
li = gidx[i] - fs;
if (pncols > 0) {
ierr = MatSetValues(*P,1,&ci,pncols,picol,pvcol,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = ISRestoreIndices(lis,&gidx);CHKERRQ(ierr);
ierr = VecRestoreArray(lv,&lcid);CHKERRQ(ierr);
ierr = PetscFree(pcontrib);CHKERRQ(ierr);
ierr = PetscFree(picol);CHKERRQ(ierr);
ierr = PetscFree(pvcol);CHKERRQ(ierr);
ierr = PetscFree(lsparse);CHKERRQ(ierr);
ierr = PetscFree(gsparse);CHKERRQ(ierr);
ierr = ISDestroy(&lis);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lA);CHKERRQ(ierr);
ierr = VecDestroy(&lv);CHKERRQ(ierr);
ierr = VecDestroy(&cv);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
ierr = VecScatterDestroy(&lscat);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Mat Pold;
ierr = PCGAMGProlongator_Classical(pc,A,G,agg_lists,&Pold);CHKERRQ(ierr);
ierr = MatView(Pold,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatView(*P,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&Pold);CHKERRQ(ierr);
*/
PetscFunctionReturn(0);
}
void PETSC_STDCALL issort_(IS is, int *__ierr ){
*__ierr = ISSort(
(IS)PetscToPointer((is) ));
}
int main(int argc,char **args)
{
Mat A, *S;
IS rowis[2], colis[2];
PetscInt n,N,i,j,k,l,nsub,Jlow[2] = {0,1}, *jlow, Jhigh[2] = {3,4}, *jhigh, row, col, *subindices, ncols;
const PetscInt *cols;
PetscScalar v;
PetscMPIInt rank, size, p, inversions, total_inversions;
PetscBool sort_rows, sort_cols, show_inversions;
PetscErrorCode ierr;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size>2) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_ARG_WRONG, "A uniprocessor or two-processor example only.\n");
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
if (size > 1) {
n = 8; N = 16;
} else {
n = 16; N = 16;
}
ierr = MatSetSizes(A,n,n,N,N);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/* Don't care if the entries are set multiple times by different procs. */
for (i=0; i<4; ++i) {
for (j = 0; j<4; ++j) {
row = j*4+i;
v = -1.0;
if (i>0) {
col = row-1; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (i<3) {
col = row+1; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (j>0) {
col = row-4; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
if (j<3) {
col = row+4; ierr = MatSetValues(A,1,&row,1,&col,&v,INSERT_VALUES);CHKERRQ(ierr);
}
v = 4.0;
ierr = MatSetValues(A,1,&row,1,&row,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Original matrix\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (size > 1) {
nsub = 1; /* one subdomain per rank */
}
else {
nsub = 2; /* both subdomains on rank 0 */
}
if (rank) {
jlow = Jlow+1; jhigh = Jhigh+1;
}
else {
jlow = Jlow; jhigh = Jhigh;
}
sort_rows = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-sort_rows", &sort_rows, NULL);CHKERRQ(ierr);
sort_cols = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-sort_cols", &sort_cols, NULL);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
ierr = PetscMalloc1(12, &subindices);CHKERRQ(ierr);
k = 0;
for (i = 0; i < 4; ++i) {
for (j = jlow[l]; j < jhigh[l]; ++j) {
subindices[k] = j*4+i;
k++;
}
}
ierr = ISCreateGeneral(PETSC_COMM_SELF, 12, subindices, PETSC_OWN_POINTER, rowis+l);CHKERRQ(ierr);
if ((sort_rows && !sort_cols) || (!sort_rows && sort_cols)) {
ierr = ISDuplicate(rowis[l],colis+l);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)rowis[l]);CHKERRQ(ierr);
colis[l] = rowis[l];
}
if (sort_rows) {
ierr = ISSort(rowis[l]);CHKERRQ(ierr);
}
if (sort_cols) {
ierr = ISSort(colis[l]);CHKERRQ(ierr);
}
}
ierr = PetscMalloc1(nsub, &S);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,nsub,rowis,colis,MAT_INITIAL_MATRIX, &S);CHKERRQ(ierr);
show_inversions = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL, "-show_inversions", &show_inversions, NULL);CHKERRQ(ierr);
inversions = 0;
for (p = 0; p < size; ++p) {
if (p == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Number of subdomains: %D:\n", rank, size, nsub);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
PetscInt i0, i1;
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Subdomain row IS %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = ISView(rowis[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Subdomain col IS %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = ISView(colis[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF, "[%D:%D]: Submatrix %D:\n", rank, size, l);CHKERRQ(ierr);
ierr = MatView(S[l],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
if (show_inversions) {
ierr = MatGetOwnershipRange(S[l], &i0,&i1);CHKERRQ(ierr);
for (i = i0; i < i1; ++i) {
ierr = MatGetRow(S[l], i, &ncols, &cols, NULL);CHKERRQ(ierr);
for (j = 1; j < ncols; ++j) {
if (cols[j] < cols[j-1]) {
ierr = PetscPrintf(PETSC_COMM_SELF, "***Inversion in row %D: col[%D] = %D < %D = col[%D]\n", i, j, cols[j], cols[j-1], j-1);CHKERRQ(ierr);
inversions++;
}
}
ierr = MatRestoreRow(S[l], i, &ncols, &cols, NULL);CHKERRQ(ierr);
}
}
}
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
if (show_inversions) {
ierr = MPI_Reduce(&inversions,&total_inversions,1,MPIU_INT, MPIU_SUM,0,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "*Total inversions: %D\n", total_inversions);CHKERRQ(ierr);
}
ierr = MatDestroy(&A);CHKERRQ(ierr);
for (l = 0; l < nsub; ++l) {
ierr = MatDestroy(&(S[l]));CHKERRQ(ierr);
ierr = ISDestroy(&(rowis[l]));CHKERRQ(ierr);
ierr = ISDestroy(&(colis[l]));CHKERRQ(ierr);
}
ierr = PetscFree(S);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
MatType mtype = MATMPIAIJ; /* matrix format */
Mat A,B; /* matrix */
PetscViewer fd; /* viewer */
char file[PETSC_MAX_PATH_LEN]; /* input file name */
PetscBool flg,viewMats,viewIS,viewVecs;
PetscInt ierr,*nlocal,m,n;
PetscMPIInt rank,size;
MatPartitioning part;
IS is,isn;
Vec xin, xout;
VecScatter scat;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_mats", &viewMats);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_is", &viewIS);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-view_vecs", &viewVecs);CHKERRQ(ierr);
/*
Determine file from which we read the matrix
*/
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
/*
Open binary file. Note that we use FILE_MODE_READ to indicate
reading from this file.
*/
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
/*
Load the matrix and vector; then destroy the viewer.
*/
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetType(A,mtype);CHKERRQ(ierr);
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&xin);CHKERRQ(ierr);
ierr = VecLoad(xin,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Original matrix:\n");
ierr = MatView(A,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
if (viewVecs) {
if (!rank) printf("Original vector:\n");
ierr = VecView(xin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Partition the graph of the matrix */
ierr = MatPartitioningCreate(PETSC_COMM_WORLD,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,A);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
/* get new processor owner number of each vertex */
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS1 - new processor ownership:\n");
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* get new global number of each old global number */
ierr = ISPartitioningToNumbering(is,&isn);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS2 - new global numbering:\n");
ierr = ISView(isn,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* get number of new vertices for each processor */
ierr = PetscMalloc(size*sizeof(PetscInt),&nlocal);CHKERRQ(ierr);
ierr = ISPartitioningCount(is,size,nlocal);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
/* get old global number of each new global number */
ierr = ISInvertPermutation(isn,nlocal[rank],&is);CHKERRQ(ierr);
ierr = PetscFree(nlocal);CHKERRQ(ierr);
ierr = ISDestroy(&isn);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
if (viewIS) {
if (!rank) printf("IS3=inv(IS2) - old global number of each new global number:\n");
ierr = ISView(is,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* move the matrix rows to the new processes they have been assigned to by the permutation */
ierr = ISSort(is);CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,is,is,MAT_INITIAL_MATRIX,&B);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
/* move the vector rows to the new processes they have been assigned to */
ierr = MatGetLocalSize(B,&m,&n);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,m,PETSC_DECIDE,&xout);CHKERRQ(ierr);
ierr = VecScatterCreate(xin,is,xout,NULL,&scat);CHKERRQ(ierr);
ierr = VecScatterBegin(scat,xin,xout,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scat,xin,xout,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scat);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Partitioned matrix:\n");
ierr = MatView(B,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
if (viewVecs) {
if (!rank) printf("Mapped vector:\n");
ierr = VecView(xout,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
{
PetscInt rstart,i,*nzd,*nzo,nzl,nzmax = 0,*ncols,nrow,j;
Mat J;
const PetscInt *cols;
const PetscScalar *vals;
PetscScalar *nvals;
ierr = MatGetOwnershipRange(B,&rstart,NULL);CHKERRQ(ierr);
ierr = PetscMalloc(2*m*sizeof(PetscInt),&nzd);CHKERRQ(ierr);
ierr = PetscMemzero(nzd,2*m*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMalloc(2*m*sizeof(PetscInt),&nzo);CHKERRQ(ierr);
ierr = PetscMemzero(nzo,2*m*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0; i<m; i++) {
ierr = MatGetRow(B,i+rstart,&nzl,&cols,NULL);CHKERRQ(ierr);
for (j=0; j<nzl; j++) {
if (cols[j] >= rstart && cols[j] < rstart+n) {
nzd[2*i] += 2;
nzd[2*i+1] += 2;
} else {
nzo[2*i] += 2;
nzo[2*i+1] += 2;
}
}
nzmax = PetscMax(nzmax,nzd[2*i]+nzo[2*i]);
ierr = MatRestoreRow(B,i+rstart,&nzl,&cols,NULL);CHKERRQ(ierr);
}
ierr = MatCreateAIJ(PETSC_COMM_WORLD,2*m,2*m,PETSC_DECIDE,PETSC_DECIDE,0,nzd,0,nzo,&J);CHKERRQ(ierr);
ierr = PetscInfo(0,"Created empty Jacobian matrix\n");CHKERRQ(ierr);
ierr = PetscFree(nzd);CHKERRQ(ierr);
ierr = PetscFree(nzo);CHKERRQ(ierr);
ierr = PetscMalloc2(nzmax,PetscInt,&ncols,nzmax,PetscScalar,&nvals);CHKERRQ(ierr);
ierr = PetscMemzero(nvals,nzmax*sizeof(PetscScalar));CHKERRQ(ierr);
for (i=0; i<m; i++) {
ierr = MatGetRow(B,i+rstart,&nzl,&cols,&vals);CHKERRQ(ierr);
for (j=0; j<nzl; j++) {
ncols[2*j] = 2*cols[j];
ncols[2*j+1] = 2*cols[j]+1;
}
nrow = 2*(i+rstart);
ierr = MatSetValues(J,1,&nrow,2*nzl,ncols,nvals,INSERT_VALUES);CHKERRQ(ierr);
nrow = 2*(i+rstart) + 1;
ierr = MatSetValues(J,1,&nrow,2*nzl,ncols,nvals,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(B,i+rstart,&nzl,&cols,&vals);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (viewMats) {
if (!rank) printf("Jacobian matrix structure:\n");
ierr = MatView(J,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr);
}
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = PetscFree2(ncols,nvals);CHKERRQ(ierr);
}
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = VecDestroy(&xin);CHKERRQ(ierr);
ierr = VecDestroy(&xout);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode MatColoringGetDegrees(Mat G,PetscInt distance,PetscInt *degrees)
{
PetscInt j,i,s,e,n,ln,lm,degree,bidx,idx,dist;
Mat lG,*lGs;
IS ris;
PetscErrorCode ierr;
PetscInt *seen;
const PetscInt *gidx;
PetscInt *idxbuf;
PetscInt *distbuf;
PetscInt ncols;
const PetscInt *cols;
PetscBool isSEQAIJ;
Mat_SeqAIJ *aij;
PetscInt *Gi,*Gj;
PetscFunctionBegin;
ierr = MatGetOwnershipRange(G,&s,&e);CHKERRQ(ierr);
n=e-s;
ierr = ISCreateStride(PetscObjectComm((PetscObject)G),n,s,1,&ris);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(G,1,&ris,distance);CHKERRQ(ierr);
ierr = ISSort(ris);CHKERRQ(ierr);
ierr = MatGetSubMatrices(G,1,&ris,&ris,MAT_INITIAL_MATRIX,&lGs);CHKERRQ(ierr);
lG = lGs[0];
ierr = PetscObjectTypeCompare((PetscObject)lG,MATSEQAIJ,&isSEQAIJ);CHKERRQ(ierr);
if (!isSEQAIJ) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_SUP,"Requires an MPI/SEQAIJ Matrix");
ierr = MatGetSize(lG,&ln,&lm);CHKERRQ(ierr);
aij = (Mat_SeqAIJ*)lG->data;
Gi = aij->i;
Gj = aij->j;
ierr = PetscMalloc3(lm,&seen,lm,&idxbuf,lm,&distbuf);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
seen[i]=-1;
}
ierr = ISGetIndices(ris,&gidx);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
if (gidx[i] >= e || gidx[i] < s) continue;
bidx=-1;
ncols = Gi[i+1]-Gi[i];
cols = &(Gj[Gi[i]]);
degree = 0;
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
degree++;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
degrees[gidx[i]-s] = degree;
}
ierr = ISRestoreIndices(ris,&gidx);CHKERRQ(ierr);
ierr = ISDestroy(&ris);CHKERRQ(ierr);
ierr = PetscFree3(seen,idxbuf,distbuf);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lGs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatColoringCreateSmallestLastWeights(MatColoring mc,PetscReal *weights)
{
PetscInt *degrees,*degb,*llprev,*llnext;
PetscInt j,i,s,e,n,nin,ln,lm,degree,maxdegree=0,bidx,idx,dist,distance=mc->dist;
Mat lG,*lGs;
IS ris;
PetscErrorCode ierr;
PetscInt *seen;
const PetscInt *gidx;
PetscInt *idxbuf;
PetscInt *distbuf;
PetscInt ncols,nxt,prv,cur;
const PetscInt *cols;
PetscBool isSEQAIJ;
Mat_SeqAIJ *aij;
PetscInt *Gi,*Gj,*rperm;
Mat G = mc->mat;
PetscReal *lweights,r;
PetscRandom rand;
PetscFunctionBegin;
ierr = MatGetOwnershipRange(G,&s,&e);CHKERRQ(ierr);
n=e-s;
ierr = ISCreateStride(PetscObjectComm((PetscObject)G),n,s,1,&ris);CHKERRQ(ierr);
ierr = MatIncreaseOverlap(G,1,&ris,distance+1);CHKERRQ(ierr);
ierr = ISSort(ris);CHKERRQ(ierr);
ierr = MatGetSubMatrices(G,1,&ris,&ris,MAT_INITIAL_MATRIX,&lGs);CHKERRQ(ierr);
lG = lGs[0];
ierr = PetscObjectTypeCompare((PetscObject)lG,MATSEQAIJ,&isSEQAIJ);CHKERRQ(ierr);
if (!isSEQAIJ) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_ARG_WRONGSTATE,"Requires an MPI/SEQAIJ Matrix");
ierr = MatGetSize(lG,&ln,&lm);CHKERRQ(ierr);
aij = (Mat_SeqAIJ*)lG->data;
Gi = aij->i;
Gj = aij->j;
ierr = PetscMalloc3(lm,&seen,lm,&idxbuf,lm,&distbuf);CHKERRQ(ierr);
ierr = PetscMalloc1(lm,°rees);CHKERRQ(ierr);
ierr = PetscMalloc1(lm,&lweights);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
seen[i]=-1;
lweights[i] = 1.;
}
ierr = ISGetIndices(ris,&gidx);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
bidx=-1;
ncols = Gi[i+1]-Gi[i];
cols = &(Gj[Gi[i]]);
degree = 0;
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
degree++;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
degrees[i] = degree;
if (degree > maxdegree) maxdegree = degree;
}
/* bucket by degree by some random permutation */
ierr = PetscRandomCreate(PetscObjectComm((PetscObject)mc),&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = PetscMalloc1(ln,&rperm);CHKERRQ(ierr);
for (i=0;i<ln;i++) {
ierr = PetscRandomGetValueReal(rand,&r);CHKERRQ(ierr);
lweights[i] = r;
rperm[i]=i;
}
ierr = PetscSortRealWithPermutation(lm,lweights,rperm);CHKERRQ(ierr);
ierr = PetscMalloc1(maxdegree+1,°b);CHKERRQ(ierr);
ierr = PetscMalloc2(ln,&llnext,ln,&llprev);CHKERRQ(ierr);
for (i=0;i<maxdegree+1;i++) {
degb[i] = -1;
}
for (i=0;i<ln;i++) {
llnext[i] = -1;
llprev[i] = -1;
seen[i] = -1;
}
for (i=0;i<ln;i++) {
idx = rperm[i];
llnext[idx] = degb[degrees[idx]];
if (degb[degrees[idx]] > 0) llprev[degb[degrees[idx]]] = idx;
degb[degrees[idx]] = idx;
}
ierr = PetscFree(rperm);CHKERRQ(ierr);
/* remove the lowest degree one */
i=0;
nin=0;
while (i != maxdegree+1) {
for (i=1;i<maxdegree+1; i++) {
if (degb[i] > 0) {
cur = degb[i];
nin++;
degrees[cur] = 0;
degb[i] = llnext[cur];
bidx=-1;
ncols = Gi[cur+1]-Gi[cur];
cols = &(Gj[Gi[cur]]);
/* place the distance-one neighbors on the queue */
for (j=0;j<ncols;j++) {
if (cols[j] != cur) {
bidx++;
seen[cols[j]] = i;
distbuf[bidx] = 1;
idxbuf[bidx] = cols[j];
}
}
while (bidx >= 0) {
/* pop */
idx = idxbuf[bidx];
dist = distbuf[bidx];
bidx--;
nxt=llnext[idx];
prv=llprev[idx];
if (degrees[idx] > 0) {
/* change up the degree of the neighbors still in the graph */
if (lweights[idx] <= lweights[cur]) lweights[idx] = lweights[cur]+1;
if (nxt > 0) {
llprev[nxt] = prv;
}
if (prv > 0) {
llnext[prv] = nxt;
} else {
degb[degrees[idx]] = nxt;
}
degrees[idx]--;
llnext[idx] = degb[degrees[idx]];
llprev[idx] = -1;
if (degb[degrees[idx]] >= 0) {
llprev[degb[degrees[idx]]] = idx;
}
degb[degrees[idx]] = idx;
if (dist < distance) {
ncols = Gi[idx+1]-Gi[idx];
cols = &(Gj[Gi[idx]]);
for (j=0;j<ncols;j++) {
if (seen[cols[j]] != i) {
bidx++;
seen[cols[j]] = i;
idxbuf[bidx] = cols[j];
distbuf[bidx] = dist+1;
}
}
}
}
}
break;
}
}
}
for (i=0;i<lm;i++) {
if (gidx[i] >= s && gidx[i] < e) {
weights[gidx[i]-s] = lweights[i];
}
}
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFree(degb);CHKERRQ(ierr);
ierr = PetscFree2(llnext,llprev);CHKERRQ(ierr);
ierr = PetscFree(degrees);CHKERRQ(ierr);
ierr = PetscFree(lweights);CHKERRQ(ierr);
ierr = ISRestoreIndices(ris,&gidx);CHKERRQ(ierr);
ierr = ISDestroy(&ris);CHKERRQ(ierr);
ierr = PetscFree3(seen,idxbuf,distbuf);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lGs);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCSubSchursSetUp(PCBDDCSubSchurs sub_schurs, Mat S, IS is_A_I, IS is_A_B, PetscInt ncc, IS is_cc[], PetscInt xadj[], PetscInt adjncy[], PetscInt nlayers)
{
Mat A_II,A_IB,A_BI,A_BB;
ISLocalToGlobalMapping BtoNmap,ItoNmap;
PetscBT touched;
PetscInt i,n_I,n_B,n_local,*local_numbering;
PetscBool is_sorted;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = ISSorted(is_A_I,&is_sorted);CHKERRQ(ierr);
if (!is_sorted) {
SETERRQ(PetscObjectComm((PetscObject)is_A_I),PETSC_ERR_PLIB,"IS for I dofs should be shorted");
}
ierr = ISSorted(is_A_B,&is_sorted);CHKERRQ(ierr);
if (!is_sorted) {
SETERRQ(PetscObjectComm((PetscObject)is_A_B),PETSC_ERR_PLIB,"IS for B dofs should be shorted");
}
/* get sizes */
ierr = ISGetLocalSize(is_A_I,&n_I);CHKERRQ(ierr);
ierr = ISGetLocalSize(is_A_B,&n_B);CHKERRQ(ierr);
n_local = n_I+n_B;
/* maps */
ierr = ISLocalToGlobalMappingCreateIS(is_A_B,&BtoNmap);CHKERRQ(ierr);
if (nlayers >= 0 && xadj != NULL && adjncy != NULL) { /* I problems have a different size of the original ones */
ierr = ISLocalToGlobalMappingCreateIS(is_A_I,&ItoNmap);CHKERRQ(ierr);
/* allocate some auxiliary space */
ierr = PetscMalloc1(n_local,&local_numbering);CHKERRQ(ierr);
ierr = PetscBTCreate(n_local,&touched);CHKERRQ(ierr);
} else {
ItoNmap = 0;
local_numbering = 0;
touched = 0;
}
/* get Schur complement matrices */
ierr = MatSchurComplementGetSubMatrices(S,&A_II,NULL,&A_IB,&A_BI,&A_BB);CHKERRQ(ierr);
/* allocate space for schur complements */
ierr = PetscMalloc5(ncc,&sub_schurs->is_AEj_I,ncc,&sub_schurs->is_AEj_B,ncc,&sub_schurs->S_Ej,ncc,&sub_schurs->work1,ncc,&sub_schurs->work2);CHKERRQ(ierr);
sub_schurs->n_subs = ncc;
/* cycle on subsets and extract schur complements */
for (i=0;i<sub_schurs->n_subs;i++) {
Mat AE_II,AE_IE,AE_EI,AE_EE;
IS is_I,is_subset_B;
/* get IS for subsets in B numbering */
ierr = ISDuplicate(is_cc[i],&sub_schurs->is_AEj_B[i]);CHKERRQ(ierr);
ierr = ISSort(sub_schurs->is_AEj_B[i]);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyIS(BtoNmap,IS_GTOLM_DROP,sub_schurs->is_AEj_B[i],&is_subset_B);CHKERRQ(ierr);
/* BB block on subset */
ierr = MatGetSubMatrix(A_BB,is_subset_B,is_subset_B,MAT_INITIAL_MATRIX,&AE_EE);CHKERRQ(ierr);
if (ItoNmap) { /* is ItoNmap has been computed, extracts only a part of I dofs */
const PetscInt* idx_B;
PetscInt n_local_dofs,n_prev_added,j,layer,subset_size;
/* all boundary dofs must be skipped when adding layers */
ierr = PetscBTMemzero(n_local,touched);CHKERRQ(ierr);
ierr = ISGetIndices(is_A_B,&idx_B);CHKERRQ(ierr);
for (j=0;j<n_B;j++) {
ierr = PetscBTSet(touched,idx_B[j]);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(is_A_B,&idx_B);CHKERRQ(ierr);
/* add next layers of dofs */
ierr = ISGetLocalSize(is_cc[i],&subset_size);CHKERRQ(ierr);
ierr = ISGetIndices(is_cc[i],&idx_B);CHKERRQ(ierr);
ierr = PetscMemcpy(local_numbering,idx_B,subset_size*sizeof(PetscInt));CHKERRQ(ierr);
ierr = ISRestoreIndices(is_cc[i],&idx_B);CHKERRQ(ierr);
n_local_dofs = subset_size;
n_prev_added = subset_size;
for (layer=0;layer<nlayers;layer++) {
PetscInt n_added;
if (n_local_dofs == n_I+subset_size) break;
if (n_local_dofs > n_I+subset_size) {
SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Error querying layer %d. Out of bound access (%d > %d)",layer,n_local_dofs,n_I+subset_size);
}
ierr = PCBDDCAdjGetNextLayer_Private(local_numbering+n_local_dofs,n_prev_added,touched,xadj,adjncy,&n_added);CHKERRQ(ierr);
n_prev_added = n_added;
n_local_dofs += n_added;
if (!n_added) break;
}
/* IS for I dofs in original numbering and in I numbering */
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)ItoNmap),n_local_dofs-subset_size,local_numbering+subset_size,PETSC_COPY_VALUES,&sub_schurs->is_AEj_I[i]);CHKERRQ(ierr);
ierr = ISSort(sub_schurs->is_AEj_I[i]);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyIS(ItoNmap,IS_GTOLM_DROP,sub_schurs->is_AEj_I[i],&is_I);CHKERRQ(ierr);
/* II block */
ierr = MatGetSubMatrix(A_II,is_I,is_I,MAT_INITIAL_MATRIX,&AE_II);CHKERRQ(ierr);
} else { /* in this case we can take references of already existing IS and matrices for I dofs */
/* IS for I dofs in original numbering */
ierr = PetscObjectReference((PetscObject)is_A_I);CHKERRQ(ierr);
sub_schurs->is_AEj_I[i] = is_A_I;
/* IS for I dofs in I numbering TODO: "first" argument of ISCreateStride is not general */
ierr = ISCreateStride(PetscObjectComm((PetscObject)is_A_I),n_I,0,1,&is_I);CHKERRQ(ierr);
/* II block is the same */
ierr = PetscObjectReference((PetscObject)A_II);CHKERRQ(ierr);
AE_II = A_II;
}
/* IE block */
ierr = MatGetSubMatrix(A_IB,is_I,is_subset_B,MAT_INITIAL_MATRIX,&AE_IE);CHKERRQ(ierr);
/* EI block */
ierr = MatGetSubMatrix(A_BI,is_subset_B,is_I,MAT_INITIAL_MATRIX,&AE_EI);CHKERRQ(ierr);
/* setup Schur complements on subset */
ierr = MatCreateSchurComplement(AE_II,AE_II,AE_IE,AE_EI,AE_EE,&sub_schurs->S_Ej[i]);CHKERRQ(ierr);
ierr = MatGetVecs(sub_schurs->S_Ej[i],&sub_schurs->work1[i],&sub_schurs->work2[i]);CHKERRQ(ierr);
if (AE_II == A_II) { /* we can reuse the same ksp */
KSP ksp;
ierr = MatSchurComplementGetKSP(S,&ksp);CHKERRQ(ierr);
ierr = MatSchurComplementSetKSP(sub_schurs->S_Ej[i],ksp);CHKERRQ(ierr);
} else { /* build new ksp object which inherits ksp and pc types from the original one */
KSP origksp,schurksp;
PC origpc,schurpc;
KSPType ksp_type;
PCType pc_type;
PetscInt n_internal;
ierr = MatSchurComplementGetKSP(S,&origksp);CHKERRQ(ierr);
ierr = MatSchurComplementGetKSP(sub_schurs->S_Ej[i],&schurksp);CHKERRQ(ierr);
ierr = KSPGetType(origksp,&ksp_type);CHKERRQ(ierr);
ierr = KSPSetType(schurksp,ksp_type);CHKERRQ(ierr);
ierr = KSPGetPC(schurksp,&schurpc);CHKERRQ(ierr);
ierr = KSPGetPC(origksp,&origpc);CHKERRQ(ierr);
ierr = PCGetType(origpc,&pc_type);CHKERRQ(ierr);
ierr = PCSetType(schurpc,pc_type);CHKERRQ(ierr);
ierr = ISGetSize(is_I,&n_internal);CHKERRQ(ierr);
if (n_internal) { /* UMFPACK gives error with 0 sized problems */
MatSolverPackage solver=NULL;
ierr = PCFactorGetMatSolverPackage(origpc,(const MatSolverPackage*)&solver);CHKERRQ(ierr);
if (solver) {
ierr = PCFactorSetMatSolverPackage(schurpc,solver);CHKERRQ(ierr);
}
}
ierr = KSPSetUp(schurksp);CHKERRQ(ierr);
}
/* free */
ierr = MatDestroy(&AE_II);CHKERRQ(ierr);
ierr = MatDestroy(&AE_EE);CHKERRQ(ierr);
ierr = MatDestroy(&AE_IE);CHKERRQ(ierr);
ierr = MatDestroy(&AE_EI);CHKERRQ(ierr);
ierr = ISDestroy(&is_I);CHKERRQ(ierr);
ierr = ISDestroy(&is_subset_B);CHKERRQ(ierr);
}
/* free */
ierr = ISLocalToGlobalMappingDestroy(&ItoNmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&BtoNmap);CHKERRQ(ierr);
ierr = PetscFree(local_numbering);CHKERRQ(ierr);
ierr = PetscBTDestroy(&touched);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode permutematrix(Mat Ain, Mat Bin, Mat *Aout, Mat *Bout, int **permIndices)
{
PetscErrorCode ierr;
MatPartitioning part;
IS isn, is, iscols;
PetscInt *nlocal,localCols,m,n;
PetscMPIInt size, rank;
MPI_Comm comm;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Ain,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MatGetSize(Ain,&m,&n);CHKERRQ(ierr);
ierr = MatPartitioningCreate(comm,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,Ain);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
/* get new processor owner number of each vertex */
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
/* get new global number of each old global number */
ierr = ISPartitioningToNumbering(is,&isn);CHKERRQ(ierr);
ierr = PetscMalloc(size*sizeof(int),&nlocal);CHKERRQ(ierr);
/* get number of new vertices for each processor */
ierr = ISPartitioningCount(is,size,nlocal);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
/* get old global number of each new global number */
ierr = ISInvertPermutation(isn,nlocal[rank],&is);CHKERRQ(ierr);
ierr = ISDestroy(&isn);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = ISSort(is);CHKERRQ(ierr);
/* If matrix is square, the permutation is applied to rows and columns;
otherwise it is only applied to rows. */
if (m == n) {
iscols = is;
localCols = nlocal[rank];
} else {
PetscInt lowj, highj;
ierr = MatGetOwnershipRangeColumn(Ain,&lowj,&highj);CHKERRQ(ierr);
localCols = highj-lowj;
ierr = ISCreateStride(comm,localCols, lowj, 1, &iscols);CHKERRQ(ierr);
}
/* copy permutation */
if (permIndices) {
const PetscInt *indices;
PetscInt i;
*permIndices = malloc(sizeof(int)*(nlocal[rank]+localCols));
ierr = ISGetIndices(is, &indices);CHKERRQ(ierr);
for (i=0; i<nlocal[rank]; i++) (*permIndices)[i] = indices[i];
ierr = ISRestoreIndices(is, &indices);CHKERRQ(ierr);
ierr = ISGetIndices(iscols, &indices);CHKERRQ(ierr);
for (i=0; i<localCols; i++) (*permIndices)[i+nlocal[rank]] = indices[i];
ierr = ISRestoreIndices(iscols, &indices);CHKERRQ(ierr);
}
ierr = PetscFree(nlocal);CHKERRQ(ierr);
ierr = MatGetSubMatrix(Ain,is,iscols,MAT_INITIAL_MATRIX,Aout);CHKERRQ(ierr);
if (Bin && Bout) {
ierr = MatGetSubMatrix(Bin,is,iscols,MAT_INITIAL_MATRIX,Bout);CHKERRQ(ierr);
}
ierr = ISDestroy(&is);CHKERRQ(ierr);
if (m != n) {
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
