PetscErrorCode MatSetValues_Preallocator(Mat A, PetscInt m, const PetscInt *rows, PetscInt n, const PetscInt *cols, const PetscScalar *values, InsertMode addv)
{
Mat_Preallocator *p = (Mat_Preallocator *) A->data;
PetscInt rStart, rEnd, r, cStart, cEnd, c;
PetscErrorCode ierr;
PetscFunctionBegin;
/* TODO: Handle blocksize */
ierr = MatGetOwnershipRange(A, &rStart, &rEnd);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(A, &cStart, &cEnd);CHKERRQ(ierr);
for (r = 0; r < m; ++r) {
PetscHashIJKey key;
PetscBool missing;
key.i = rows[r];
if (key.i < 0) continue;
if ((key.i < rStart) || (key.i >= rEnd)) {
ierr = MatStashValuesRow_Private(&A->stash, key.i, n, cols, values, PETSC_FALSE);CHKERRQ(ierr);
} else {
for (c = 0; c < n; ++c) {
key.j = cols[c];
if (key.j < 0) continue;
ierr = PetscHSetIJQueryAdd(p->ht, key, &missing);CHKERRQ(ierr);
if (missing) {
if ((key.j >= cStart) && (key.j < cEnd)) ++p->dnz[key.i-rStart];
else ++p->onz[key.i-rStart];
}
}
}
}
PetscFunctionReturn(0);
}
PetscErrorCode PCGAMGOptProl_Classical_Jacobi(PC pc,const Mat A,Mat *P)
{
PetscErrorCode ierr;
PetscInt f,s,n,cf,cs,i,idx;
PetscInt *coarserows;
PetscInt ncols;
const PetscInt *pcols;
const PetscScalar *pvals;
Mat Pnew;
Vec diag;
PC_MG *mg = (PC_MG*)pc->data;
PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
PC_GAMG_Classical *cls = (PC_GAMG_Classical*)pc_gamg->subctx;
PetscFunctionBegin;
if (cls->nsmooths == 0) {
ierr = PCGAMGTruncateProlongator_Private(pc,P);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
ierr = MatGetOwnershipRange(*P,&s,&f);CHKERRQ(ierr);
n = f-s;
ierr = MatGetOwnershipRangeColumn(*P,&cs,&cf);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*n,&coarserows);CHKERRQ(ierr);
/* identify the rows corresponding to coarse unknowns */
idx = 0;
for (i=s;i<f;i++) {
ierr = MatGetRow(*P,i,&ncols,&pcols,&pvals);CHKERRQ(ierr);
/* assume, for now, that it's a coarse unknown if it has a single unit entry */
if (ncols == 1) {
if (pvals[0] == 1.) {
coarserows[idx] = i;
idx++;
}
}
ierr = MatRestoreRow(*P,i,&ncols,&pcols,&pvals);CHKERRQ(ierr);
}
ierr = MatGetVecs(A,&diag,0);CHKERRQ(ierr);
ierr = MatGetDiagonal(A,diag);CHKERRQ(ierr);
ierr = VecReciprocal(diag);CHKERRQ(ierr);
for (i=0;i<cls->nsmooths;i++) {
ierr = MatMatMult(A,*P,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&Pnew);CHKERRQ(ierr);
ierr = MatZeroRows(Pnew,idx,coarserows,0.,NULL,NULL);CHKERRQ(ierr);
ierr = MatDiagonalScale(Pnew,diag,0);CHKERRQ(ierr);
ierr = MatAYPX(Pnew,-1.0,*P,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatDestroy(P);CHKERRQ(ierr);
*P = Pnew;
Pnew = NULL;
}
ierr = VecDestroy(&diag);CHKERRQ(ierr);
ierr = PetscFree(coarserows);CHKERRQ(ierr);
ierr = PCGAMGTruncateProlongator_Private(pc,P);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode getSumSquares(Mat matrix, double *diag) {
PetscErrorCode ierr;
int i, j;
double *sumr, *sumc;
PetscInt n, mLocal, nLocal, low, high;
PetscReal *aux;
PetscFunctionBegin;
ierr = MatGetSize(matrix, NULL, &n); CHKERRQ(ierr);
ierr = MatGetLocalSize(matrix, &mLocal, &nLocal); CHKERRQ(ierr);
sumr = diag; sumc = &diag[mLocal];
ierr = PetscMalloc1(n, &aux); CHKERRQ(ierr);
ierr = MatGetColumnNorms(matrix, NORM_2, aux); CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(matrix, &low, &high);CHKERRQ(ierr);
for (i=low; i<high; i++) {
sumc[i-low] = aux[i]*aux[i];
}
ierr = PetscFree(aux); CHKERRQ(ierr);
ierr = MatGetOwnershipRange(matrix, &low, &high); CHKERRQ(ierr);
for (i=low; i<high; i++) {
PetscInt ncols;
const PetscInt *cols;
const PetscScalar *vals;
sumr[i-low] = 0.0;
ierr = MatGetRow(matrix, i, &ncols, &cols, &vals); CHKERRQ(ierr);
for (j = 0; j < ncols; j++) {
sumr[i-low] += PetscRealPart(vals[j]*PetscConj(vals[j]));
}
ierr = MatRestoreRow(matrix, i, &ncols, &cols, &vals); CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode PCApply_Kaczmarz(PC pc,Vec x,Vec y)
{
PC_Kaczmarz *jac = (PC_Kaczmarz*)pc->data;
PetscInt xs,xe,ys,ye,ncols,i,j;
const PetscInt *cols;
const PetscScalar *vals,*xarray;
PetscErrorCode ierr;
PetscScalar r;
PetscReal anrm;
PetscScalar *yarray;
PetscReal lambda=jac->lambda;
PetscFunctionBegin;
ierr = MatGetOwnershipRange(pc->pmat,&xs,&xe);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(pc->pmat,&ys,&ye);CHKERRQ(ierr);
ierr = VecSet(y,0.);CHKERRQ(ierr);
ierr = VecGetArrayRead(x,&xarray);CHKERRQ(ierr);
ierr = VecGetArray(y,&yarray);CHKERRQ(ierr);
for (i=xs;i<xe;i++) {
/* get the maximum row width and row norms */
ierr = MatGetRow(pc->pmat,i,&ncols,&cols,&vals);CHKERRQ(ierr);
r = xarray[i-xs];
anrm = 0.;
for (j=0;j<ncols;j++) {
if (cols[j] >= ys && cols[j] < ye) {
r -= yarray[cols[j]-ys]*vals[j];
}
anrm += PetscRealPart(PetscSqr(vals[j]));
}
if (anrm > 0.) {
for (j=0;j<ncols;j++) {
if (cols[j] >= ys && cols[j] < ye) {
yarray[cols[j]-ys] += vals[j]*lambda*r/anrm;
}
}
}
ierr = MatRestoreRow(pc->pmat,i,&ncols,&cols,&vals);CHKERRQ(ierr);
}
if (jac->symmetric) {
for (i=xe-1;i>=xs;i--) {
ierr = MatGetRow(pc->pmat,i,&ncols,&cols,&vals);CHKERRQ(ierr);
r = xarray[i-xs];
anrm = 0.;
for (j=0;j<ncols;j++) {
if (cols[j] >= ys && cols[j] < ye) {
r -= yarray[cols[j]-ys]*vals[j];
}
anrm += PetscRealPart(PetscSqr(vals[j]));
}
if (anrm > 0.) {
for (j=0;j<ncols;j++) {
if (cols[j] >= ys && cols[j] < ye) {
yarray[cols[j]-ys] += vals[j]*lambda*r/anrm;
}
}
}
ierr = MatRestoreRow(pc->pmat,i,&ncols,&cols,&vals);CHKERRQ(ierr);
}
}
ierr = VecRestoreArray(y,&yarray);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(x,&xarray);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
MatConvert_Basic - Converts from any input format to another format. For
parallel formats, the new matrix distribution is determined by PETSc.
Does not do preallocation so in general will be slow
*/
PetscErrorCode MatConvert_Basic(Mat mat, MatType newtype,MatReuse reuse,Mat *newmat)
{
Mat M;
const PetscScalar *vwork;
PetscErrorCode ierr;
PetscInt i,j,nz,m,n,rstart,rend,lm,ln,prbs,pcbs,cstart,cend,*dnz,*onz;
const PetscInt *cwork;
PetscBool isseqsbaij,ismpisbaij,isseqbaij,ismpibaij,isseqdense,ismpidense;
PetscFunctionBegin;
ierr = MatGetSize(mat,&m,&n);CHKERRQ(ierr);
ierr = MatGetLocalSize(mat,&lm,&ln);CHKERRQ(ierr);
if (ln == n) ln = PETSC_DECIDE; /* try to preserve column ownership */
ierr = MatCreate(PetscObjectComm((PetscObject)mat),&M);CHKERRQ(ierr);
ierr = MatSetSizes(M,lm,ln,m,n);CHKERRQ(ierr);
ierr = MatSetBlockSizes(M,mat->rmap->bs,mat->cmap->bs);CHKERRQ(ierr);
ierr = MatSetType(M,newtype);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(mat,&rstart,&rend);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATSEQSBAIJ,&isseqsbaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATMPISBAIJ,&ismpisbaij);CHKERRQ(ierr);
if (isseqsbaij || ismpisbaij) {ierr = MatSetOption(M,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);}
ierr = PetscObjectTypeCompare((PetscObject)M,MATSEQBAIJ,&isseqbaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATMPIBAIJ,&ismpibaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATSEQDENSE,&isseqdense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)M,MATMPIDENSE,&ismpidense);CHKERRQ(ierr);
if (isseqdense) {
ierr = MatSeqDenseSetPreallocation(M,NULL);CHKERRQ(ierr);
} else if (ismpidense) {
ierr = MatMPIDenseSetPreallocation(M,NULL);CHKERRQ(ierr);
} else {
/* Preallocation block sizes. (S)BAIJ matrices will have one index per block. */
prbs = (isseqbaij || ismpibaij || isseqsbaij || ismpisbaij) ? M->rmap->bs : 1;
pcbs = (isseqbaij || ismpibaij || isseqsbaij || ismpisbaij) ? M->cmap->bs : 1;
ierr = PetscMalloc2(lm/prbs,&dnz,lm/prbs,&onz);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(mat,&cstart,&cend);CHKERRQ(ierr);
for (i=0; i<lm; i+=prbs) {
ierr = MatGetRow(mat,rstart+i,&nz,&cwork,NULL);CHKERRQ(ierr);
dnz[i] = 0;
onz[i] = 0;
for (j=0; j<nz; j+=pcbs) {
if ((isseqsbaij || ismpisbaij) && cwork[j] < rstart+i) continue;
if (cstart <= cwork[j] && cwork[j] < cend) dnz[i]++;
else onz[i]++;
}
ierr = MatRestoreRow(mat,rstart+i,&nz,&cwork,NULL);CHKERRQ(ierr);
}
ierr = MatXAIJSetPreallocation(M,M->rmap->bs,dnz,onz,dnz,onz);CHKERRQ(ierr);
ierr = PetscFree2(dnz,onz);CHKERRQ(ierr);
}
for (i=rstart; i<rend; i++) {
ierr = MatGetRow(mat,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
ierr = MatSetValues(M,1,&i,nz,cwork,vwork,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(mat,i,&nz,&cwork,&vwork);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (reuse == MAT_REUSE_MATRIX) {
ierr = MatHeaderReplace(mat,M);CHKERRQ(ierr);
} else {
*newmat = M;
}
PetscFunctionReturn(0);
}
PetscErrorCode PCGAMGTruncateProlongator_Private(PC pc,Mat *P)
{
PetscInt j,i,ps,pf,pn,pcs,pcf,pcn,idx,cmax;
PetscErrorCode ierr;
const PetscScalar *pval;
const PetscInt *pcol;
PetscScalar *pnval;
PetscInt *pncol;
PetscInt ncols;
Mat Pnew;
PetscInt *lsparse,*gsparse;
PetscReal pmax_pos,pmax_neg,ptot_pos,ptot_neg,pthresh_pos,pthresh_neg;
PC_MG *mg = (PC_MG*)pc->data;
PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
PC_GAMG_Classical *cls = (PC_GAMG_Classical*)pc_gamg->subctx;
PetscFunctionBegin;
/* trim and rescale with reallocation */
ierr = MatGetOwnershipRange(*P,&ps,&pf);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(*P,&pcs,&pcf);CHKERRQ(ierr);
pn = pf-ps;
pcn = pcf-pcs;
ierr = PetscMalloc(sizeof(PetscInt)*pn,&lsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*pn,&gsparse);CHKERRQ(ierr);
/* allocate */
cmax = 0;
for (i=ps;i<pf;i++) {
lsparse[i-ps] = 0;
gsparse[i-ps] = 0;
ierr = MatGetRow(*P,i,&ncols,&pcol,&pval);CHKERRQ(ierr);
if (ncols > cmax) {
cmax = ncols;
}
pmax_pos = 0.;
pmax_neg = 0.;
for (j=0;j<ncols;j++) {
if (PetscRealPart(pval[j]) > pmax_pos) {
pmax_pos = PetscRealPart(pval[j]);
} else if (PetscRealPart(pval[j]) < pmax_neg) {
pmax_neg = PetscRealPart(pval[j]);
}
}
for (j=0;j<ncols;j++) {
if (PetscRealPart(pval[j]) >= pmax_pos*cls->interp_threshold || PetscRealPart(pval[j]) <= pmax_neg*cls->interp_threshold) {
if (pcol[j] >= pcs && pcol[j] < pcf) {
lsparse[i-ps]++;
} else {
gsparse[i-ps]++;
}
}
}
ierr = MatRestoreRow(*P,i,&ncols,&pcol,&pval);CHKERRQ(ierr);
}
ierr = PetscMalloc(sizeof(PetscScalar)*cmax,&pnval);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*cmax,&pncol);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)*P),&Pnew);CHKERRQ(ierr);
ierr = MatSetType(Pnew, MATAIJ);CHKERRQ(ierr);
ierr = MatSetSizes(Pnew,pn,pcn,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(Pnew,0,lsparse);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(Pnew,0,lsparse,0,gsparse);CHKERRQ(ierr);
for (i=ps;i<pf;i++) {
ierr = MatGetRow(*P,i,&ncols,&pcol,&pval);CHKERRQ(ierr);
pmax_pos = 0.;
pmax_neg = 0.;
for (j=0;j<ncols;j++) {
if (PetscRealPart(pval[j]) > pmax_pos) {
pmax_pos = PetscRealPart(pval[j]);
} else if (PetscRealPart(pval[j]) < pmax_neg) {
pmax_neg = PetscRealPart(pval[j]);
}
}
pthresh_pos = 0.;
pthresh_neg = 0.;
ptot_pos = 0.;
ptot_neg = 0.;
for (j=0;j<ncols;j++) {
if (PetscRealPart(pval[j]) >= cls->interp_threshold*pmax_pos) {
pthresh_pos += PetscRealPart(pval[j]);
} else if (PetscRealPart(pval[j]) <= cls->interp_threshold*pmax_neg) {
pthresh_neg += PetscRealPart(pval[j]);
}
if (PetscRealPart(pval[j]) > 0.) {
ptot_pos += PetscRealPart(pval[j]);
} else {
ptot_neg += PetscRealPart(pval[j]);
}
}
if (PetscAbsReal(pthresh_pos) > 0.) ptot_pos /= pthresh_pos;
if (PetscAbsReal(pthresh_neg) > 0.) ptot_neg /= pthresh_neg;
idx=0;
for (j=0;j<ncols;j++) {
if (PetscRealPart(pval[j]) >= pmax_pos*cls->interp_threshold) {
pnval[idx] = ptot_pos*pval[j];
pncol[idx] = pcol[j];
idx++;
} else if (PetscRealPart(pval[j]) <= pmax_neg*cls->interp_threshold) {
pnval[idx] = ptot_neg*pval[j];
pncol[idx] = pcol[j];
idx++;
}
}
ierr = MatRestoreRow(*P,i,&ncols,&pcol,&pval);CHKERRQ(ierr);
ierr = MatSetValues(Pnew,1,&i,idx,pncol,pnval,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(Pnew, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Pnew, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatDestroy(P);CHKERRQ(ierr);
*P = Pnew;
ierr = PetscFree(lsparse);CHKERRQ(ierr);
ierr = PetscFree(gsparse);CHKERRQ(ierr);
ierr = PetscFree(pncol);CHKERRQ(ierr);
ierr = PetscFree(pnval);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
const struct {PetscInt i,j; PetscScalar v;} entries[] = {{0,3,1.},{1,2,2.},{2,1,3.},{2,4,4.},{3,0,5.},{3,3,6.},{4,1,7.},{4,4,8.}};
const PetscInt ixrow[5] = {4,2,1,3,0},ixcol[5] = {3,2,1,4,0};
Mat A,B;
PetscErrorCode ierr;
PetscInt i,rstart,rend,cstart,cend;
IS isrow,iscol;
PetscViewer viewer,sviewer;
PetscBool view_sparse;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/* ------- Assemble matrix, --------- */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,5,5);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(A,&cstart,&cend);CHKERRQ(ierr);
for (i=0; i<(PetscInt)(sizeof(entries)/sizeof(entries[0])); i++) {
ierr = MatSetValue(A,entries[i].i,entries[i].j,entries[i].v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* ------ Prepare index sets ------ */
ierr = ISCreateGeneral(PETSC_COMM_WORLD,rend-rstart,ixrow+rstart,PETSC_USE_POINTER,&isrow);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,5,ixcol,PETSC_USE_POINTER,&iscol);CHKERRQ(ierr);
ierr = ISSetPermutation(isrow);CHKERRQ(ierr);
ierr = ISSetPermutation(iscol);CHKERRQ(ierr);
ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
view_sparse = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL, "-view_sparse", &view_sparse, PETSC_NULL);CHKERRQ(ierr);
if (!view_sparse) {
ierr = PetscViewerSetFormat(viewer,PETSC_VIEWER_ASCII_DENSE);CHKERRQ(ierr);
}
ierr = PetscViewerASCIIPrintf(viewer,"Original matrix\n");CHKERRQ(ierr);
ierr = MatView(A,viewer);CHKERRQ(ierr);
ierr = MatPermute(A,isrow,iscol,&B);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Permuted matrix\n");CHKERRQ(ierr);
ierr = MatView(B,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Row permutation\n");CHKERRQ(ierr);
ierr = ISView(isrow,viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Column permutation\n");CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(viewer,&sviewer);CHKERRQ(ierr);
ierr = ISView(iscol,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(viewer,&sviewer);CHKERRQ(ierr);
/* Free data structures */
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PETSC_EXTERN PetscErrorCode MatColoringCreateBipartiteGraph(MatColoring mc,PetscSF *etoc,PetscSF *etor)
{
PetscErrorCode ierr;
PetscInt nentries,ncolentries,idx;
PetscInt i,j,rs,re,cs,ce,cn;
PetscInt *rowleaf,*colleaf,*rowdata;
PetscInt ncol;
const PetscScalar *vcol;
const PetscInt *icol;
const PetscInt *coldegrees,*rowdegrees;
Mat m = mc->mat;
PetscFunctionBegin;
ierr = MatGetOwnershipRange(m,&rs,&re);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(m,&cs,&ce);CHKERRQ(ierr);
cn = ce-cs;
nentries=0;
for (i=rs;i<re;i++) {
ierr = MatGetRow(m,i,&ncol,NULL,&vcol);CHKERRQ(ierr);
for (j=0;j<ncol;j++) {
nentries++;
}
ierr = MatRestoreRow(m,i,&ncol,NULL,&vcol);CHKERRQ(ierr);
}
ierr = PetscMalloc(sizeof(PetscInt)*nentries,&rowleaf);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*nentries,&rowdata);CHKERRQ(ierr);
idx=0;
for (i=rs;i<re;i++) {
ierr = MatGetRow(m,i,&ncol,&icol,&vcol);CHKERRQ(ierr);
for (j=0;j<ncol;j++) {
rowleaf[idx] = icol[j];
rowdata[idx] = i;
idx++;
}
ierr = MatRestoreRow(m,i,&ncol,&icol,&vcol);CHKERRQ(ierr);
}
if (idx != nentries) SETERRQ2(PetscObjectComm((PetscObject)m),PETSC_ERR_NOT_CONVERGED,"Bad number of entries %d vs %d",idx,nentries);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)m),etoc);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)m),etor);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(*etoc,m->cmap,nentries,NULL,PETSC_COPY_VALUES,rowleaf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(*etoc);CHKERRQ(ierr);
/* determine the number of entries in the column matrix */
ierr = PetscLogEventBegin(Mat_Coloring_Comm,*etoc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeBegin(*etoc,&coldegrees);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(*etoc,&coldegrees);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,*etoc,0,0,0);CHKERRQ(ierr);
ncolentries=0;
for (i=0;i<cn;i++) {
ncolentries += coldegrees[i];
}
ierr = PetscMalloc(sizeof(PetscInt)*ncolentries,&colleaf);CHKERRQ(ierr);
/* create the one going the other way by building the leaf set */
ierr = PetscLogEventBegin(Mat_Coloring_Comm,*etoc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFGatherBegin(*etoc,MPIU_INT,rowdata,colleaf);CHKERRQ(ierr);
ierr = PetscSFGatherEnd(*etoc,MPIU_INT,rowdata,colleaf);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,*etoc,0,0,0);CHKERRQ(ierr);
/* this one takes mat entries in *columns* to rows -- you never have to actually be able to order the leaf entries. */
ierr = PetscSFSetGraphLayout(*etor,m->rmap,ncolentries,NULL,PETSC_COPY_VALUES,colleaf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(*etor);CHKERRQ(ierr);
ierr = PetscLogEventBegin(Mat_Coloring_Comm,*etor,0,0,0);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeBegin(*etor,&rowdegrees);CHKERRQ(ierr);
ierr = PetscSFComputeDegreeEnd(*etor,&rowdegrees);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,*etor,0,0,0);CHKERRQ(ierr);
ierr = PetscFree(rowdata);CHKERRQ(ierr);
ierr = PetscFree(rowleaf);CHKERRQ(ierr);
ierr = PetscFree(colleaf);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode SNESSetUp_Multiblock(SNES snes)
{
SNES_Multiblock *mb = (SNES_Multiblock *) snes->data;
BlockDesc blocks;
PetscInt i, numBlocks;
PetscErrorCode ierr;
PetscFunctionBegin;
/* ierr = SNESDefaultGetWork(snes, 1);CHKERRQ(ierr); */
ierr = SNESMultiblockSetDefaults(snes);CHKERRQ(ierr);
numBlocks = mb->numBlocks;
blocks = mb->blocks;
/* Create ISs */
if (!mb->issetup) {
PetscInt ccsize, rstart, rend, nslots, bs;
PetscBool sorted;
mb->issetup = PETSC_TRUE;
bs = mb->bs;
ierr = MatGetOwnershipRange(snes->jacobian_pre, &rstart, &rend);CHKERRQ(ierr);
ierr = MatGetLocalSize(snes->jacobian_pre, PETSC_NULL, &ccsize);CHKERRQ(ierr);
nslots = (rend - rstart)/bs;
for (i = 0; i < numBlocks; ++i) {
if (mb->defaultblocks) {
ierr = ISCreateStride(((PetscObject) snes)->comm, nslots, rstart+i, numBlocks, &blocks->is);CHKERRQ(ierr);
} else if (!blocks->is) {
if (blocks->nfields > 1) {
PetscInt *ii, j, k, nfields = blocks->nfields, *fields = blocks->fields;
ierr = PetscMalloc(nfields*nslots*sizeof(PetscInt), &ii);CHKERRQ(ierr);
for (j = 0; j < nslots; ++j) {
for (k = 0; k < nfields; ++k) {
ii[nfields*j + k] = rstart + bs*j + fields[k];
}
}
ierr = ISCreateGeneral(((PetscObject) snes)->comm, nslots*nfields, ii, PETSC_OWN_POINTER, &blocks->is);CHKERRQ(ierr);
} else {
ierr = ISCreateStride(((PetscObject) snes)->comm, nslots, rstart+blocks->fields[0], bs, &blocks->is);CHKERRQ(ierr);
}
}
ierr = ISSorted(blocks->is, &sorted);CHKERRQ(ierr);
if (!sorted) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_USER, "Fields must be sorted when creating split");
blocks = blocks->next;
}
}
#if 0
/* Create matrices */
ilink = jac->head;
if (!jac->pmat) {
ierr = PetscMalloc(nsplit*sizeof(Mat),&jac->pmat);CHKERRQ(ierr);
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->pmat,ilink->is,ilink->is,MAT_INITIAL_MATRIX,&jac->pmat[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
} else {
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->pmat,ilink->is,ilink->is,MAT_REUSE_MATRIX,&jac->pmat[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
}
if (jac->realdiagonal) {
ilink = jac->head;
if (!jac->mat) {
ierr = PetscMalloc(nsplit*sizeof(Mat),&jac->mat);CHKERRQ(ierr);
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->mat,ilink->is,ilink->is,MAT_INITIAL_MATRIX,&jac->mat[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
} else {
for (i=0; i<nsplit; i++) {
if (jac->mat[i]) {ierr = MatGetSubMatrix(pc->mat,ilink->is,ilink->is,MAT_REUSE_MATRIX,&jac->mat[i]);CHKERRQ(ierr);}
ilink = ilink->next;
}
}
} else {
jac->mat = jac->pmat;
}
#endif
#if 0
if (jac->type != PC_COMPOSITE_ADDITIVE && jac->type != PC_COMPOSITE_SCHUR) {
/* extract the rows of the matrix associated with each field: used for efficient computation of residual inside algorithm */
ilink = jac->head;
if (!jac->Afield) {
ierr = PetscMalloc(nsplit*sizeof(Mat),&jac->Afield);CHKERRQ(ierr);
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->mat,ilink->is,PETSC_NULL,MAT_INITIAL_MATRIX,&jac->Afield[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
} else {
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->mat,ilink->is,PETSC_NULL,MAT_REUSE_MATRIX,&jac->Afield[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
}
}
#endif
if (mb->type == PC_COMPOSITE_SCHUR) {
#if 0
IS ccis;
PetscInt rstart,rend;
if (nsplit != 2) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_ARG_INCOMP,"To use Schur complement preconditioner you must have exactly 2 fields");
/* When extracting off-diagonal submatrices, we take complements from this range */
ierr = MatGetOwnershipRangeColumn(pc->mat,&rstart,&rend);CHKERRQ(ierr);
/* need to handle case when one is resetting up the preconditioner */
if (jac->schur) {
ilink = jac->head;
ierr = ISComplement(ilink->is,rstart,rend,&ccis);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->mat,ilink->is,ccis,MAT_REUSE_MATRIX,&jac->B);CHKERRQ(ierr);
ierr = ISDestroy(&ccis);CHKERRQ(ierr);
ilink = ilink->next;
ierr = ISComplement(ilink->is,rstart,rend,&ccis);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->mat,ilink->is,ccis,MAT_REUSE_MATRIX,&jac->C);CHKERRQ(ierr);
ierr = ISDestroy(&ccis);CHKERRQ(ierr);
ierr = MatSchurComplementUpdate(jac->schur,jac->mat[0],jac->pmat[0],jac->B,jac->C,jac->pmat[1],pc->flag);CHKERRQ(ierr);
ierr = KSPSetOperators(jac->kspschur,jac->schur,FieldSplitSchurPre(jac),pc->flag);CHKERRQ(ierr);
} else {
KSP ksp;
char schurprefix[256];
/* extract the A01 and A10 matrices */
ilink = jac->head;
ierr = ISComplement(ilink->is,rstart,rend,&ccis);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->mat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->B);CHKERRQ(ierr);
ierr = ISDestroy(&ccis);CHKERRQ(ierr);
ilink = ilink->next;
ierr = ISComplement(ilink->is,rstart,rend,&ccis);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->mat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->C);CHKERRQ(ierr);
ierr = ISDestroy(&ccis);CHKERRQ(ierr);
/* Use mat[0] (diagonal block of the real matrix) preconditioned by pmat[0] */
ierr = MatCreateSchurComplement(jac->mat[0],jac->pmat[0],jac->B,jac->C,jac->mat[1],&jac->schur);CHKERRQ(ierr);
/* set tabbing and options prefix of KSP inside the MatSchur */
ierr = MatSchurComplementGetKSP(jac->schur,&ksp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)ksp,(PetscObject)pc,2);CHKERRQ(ierr);
ierr = PetscSNPrintf(schurprefix,sizeof(schurprefix),"%sfieldsplit_%s_",((PetscObject)pc)->prefix?((PetscObject)pc)->prefix:"",jac->head->splitname);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(ksp,schurprefix);CHKERRQ(ierr);
ierr = MatSetFromOptions(jac->schur);CHKERRQ(ierr);
ierr = KSPCreate(((PetscObject)pc)->comm,&jac->kspschur);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)jac->kspschur);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)jac->kspschur,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(jac->kspschur,jac->schur,FieldSplitSchurPre(jac),DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
if (jac->schurpre == PC_FIELDSPLIT_SCHUR_PRE_SELF) {
PC pc;
ierr = KSPGetPC(jac->kspschur,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE);CHKERRQ(ierr);
/* Note: This is bad if there exist preconditioners for MATSCHURCOMPLEMENT */
}
ierr = PetscSNPrintf(schurprefix,sizeof(schurprefix),"%sfieldsplit_%s_",((PetscObject)pc)->prefix?((PetscObject)pc)->prefix:"",ilink->splitname);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(jac->kspschur,schurprefix);CHKERRQ(ierr);
/* really want setfromoptions called in PCSetFromOptions_FieldSplit(), but it is not ready yet */
ierr = KSPSetFromOptions(jac->kspschur);CHKERRQ(ierr);
ierr = PetscMalloc2(2,Vec,&jac->x,2,Vec,&jac->y);CHKERRQ(ierr);
ierr = MatGetVecs(jac->pmat[0],&jac->x[0],&jac->y[0]);CHKERRQ(ierr);
ierr = MatGetVecs(jac->pmat[1],&jac->x[1],&jac->y[1]);CHKERRQ(ierr);
ilink = jac->head;
ilink->x = jac->x[0]; ilink->y = jac->y[0];
ilink = ilink->next;
ilink->x = jac->x[1]; ilink->y = jac->y[1];
}
#endif
} else {
/* Set up the individual SNESs */
blocks = mb->blocks;
i = 0;
while (blocks) {
/*TODO: Set these correctly */
/*ierr = SNESSetFunction(blocks->snes, blocks->x, func);CHKERRQ(ierr);*/
/*ierr = SNESSetJacobian(blocks->snes, blocks->x, jac);CHKERRQ(ierr);*/
ierr = VecDuplicate(blocks->snes->vec_sol, &blocks->x);CHKERRQ(ierr);
/* really want setfromoptions called in SNESSetFromOptions_Multiblock(), but it is not ready yet */
ierr = SNESSetFromOptions(blocks->snes);CHKERRQ(ierr);
ierr = SNESSetUp(blocks->snes);CHKERRQ(ierr);
blocks = blocks->next;
i++;
}
}
/* Compute scatter contexts needed by multiplicative versions and non-default splits */
if (!mb->blocks->sctx) {
Vec xtmp;
blocks = mb->blocks;
ierr = MatGetVecs(snes->jacobian_pre, &xtmp, PETSC_NULL);CHKERRQ(ierr);
while(blocks) {
ierr = VecScatterCreate(xtmp, blocks->is, blocks->x, PETSC_NULL, &blocks->sctx);CHKERRQ(ierr);
blocks = blocks->next;
}
ierr = VecDestroy(&xtmp);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode permutematrix(Mat Ain, Mat Bin, Mat *Aout, Mat *Bout, int **permIndices)
{
PetscErrorCode ierr;
MatPartitioning part;
IS isn, is, iscols;
PetscInt *nlocal,localCols,m,n;
PetscMPIInt size, rank;
MPI_Comm comm;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Ain,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MatGetSize(Ain,&m,&n);CHKERRQ(ierr);
ierr = MatPartitioningCreate(comm,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,Ain);CHKERRQ(ierr);
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
/* get new processor owner number of each vertex */
ierr = MatPartitioningApply(part,&is);CHKERRQ(ierr);
/* get new global number of each old global number */
ierr = ISPartitioningToNumbering(is,&isn);CHKERRQ(ierr);
ierr = PetscMalloc(size*sizeof(int),&nlocal);CHKERRQ(ierr);
/* get number of new vertices for each processor */
ierr = ISPartitioningCount(is,size,nlocal);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
/* get old global number of each new global number */
ierr = ISInvertPermutation(isn,nlocal[rank],&is);CHKERRQ(ierr);
ierr = ISDestroy(&isn);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = ISSort(is);CHKERRQ(ierr);
/* If matrix is square, the permutation is applied to rows and columns;
otherwise it is only applied to rows. */
if (m == n) {
iscols = is;
localCols = nlocal[rank];
} else {
PetscInt lowj, highj;
ierr = MatGetOwnershipRangeColumn(Ain,&lowj,&highj);CHKERRQ(ierr);
localCols = highj-lowj;
ierr = ISCreateStride(comm,localCols, lowj, 1, &iscols);CHKERRQ(ierr);
}
/* copy permutation */
if (permIndices) {
const PetscInt *indices;
PetscInt i;
*permIndices = malloc(sizeof(int)*(nlocal[rank]+localCols));
ierr = ISGetIndices(is, &indices);CHKERRQ(ierr);
for (i=0; i<nlocal[rank]; i++) (*permIndices)[i] = indices[i];
ierr = ISRestoreIndices(is, &indices);CHKERRQ(ierr);
ierr = ISGetIndices(iscols, &indices);CHKERRQ(ierr);
for (i=0; i<localCols; i++) (*permIndices)[i+nlocal[rank]] = indices[i];
ierr = ISRestoreIndices(iscols, &indices);CHKERRQ(ierr);
}
ierr = PetscFree(nlocal);CHKERRQ(ierr);
ierr = MatGetSubMatrix(Ain,is,iscols,MAT_INITIAL_MATRIX,Aout);CHKERRQ(ierr);
if (Bin && Bout) {
ierr = MatGetSubMatrix(Bin,is,iscols,MAT_INITIAL_MATRIX,Bout);CHKERRQ(ierr);
}
ierr = ISDestroy(&is);CHKERRQ(ierr);
if (m != n) {
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode loadmtx(const char* filename, Mat *M, PetscBool *pattern) {
PetscErrorCode ierr;
FILE *f;
MM_typecode type;
int m,n,nz,i,j,k;
PetscInt low,high,lowj,highj,*d_nz,*o_nz;
double re,im;
PetscScalar s;
long pos;
PetscFunctionBegin;
f = fopen(filename,"r");
if (!f) SETERRQ2(PETSC_COMM_SELF,1,"fopen '%s': %s",filename,strerror(errno));
/* first read to set matrix kind and size */
ierr = mm_read_banner(f,&type);CHKERRQ(ierr);
if (!mm_is_valid(type) || !mm_is_sparse(type) ||
!(mm_is_real(type) || mm_is_complex(type) || mm_is_pattern(type) || mm_is_integer(type)))
SETERRQ1(PETSC_COMM_SELF,1,"Matrix format '%s' not supported",mm_typecode_to_str(type));
#if !defined(PETSC_USE_COMPLEX)
if (mm_is_complex(type)) SETERRQ(PETSC_COMM_SELF,1,"Complex matrix not supported in real configuration");
#endif
if (pattern) *pattern = mm_is_pattern(type) ? PETSC_TRUE : PETSC_FALSE;
ierr = mm_read_mtx_crd_size(f,&m,&n,&nz);CHKERRQ(ierr);
pos = ftell(f);
ierr = MatCreate(PETSC_COMM_WORLD,M);CHKERRQ(ierr);
ierr = MatSetSizes(*M,PETSC_DECIDE,PETSC_DECIDE,(PetscInt)m,(PetscInt)n);CHKERRQ(ierr);
ierr = MatSetFromOptions(*M);CHKERRQ(ierr);
ierr = MatSetUp(*M);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(*M,&low,&high);CHKERRQ(ierr);
ierr = MatGetOwnershipRangeColumn(*M,&lowj,&highj);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*(high-low),&d_nz);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*(high-low),&o_nz);CHKERRQ(ierr);
for (i=0; i<high-low;i++) {
d_nz[i] = (i+low>=lowj && i+low<highj) ? 1 : 0;
o_nz[i] = (i+low>=lowj && i+low<highj) ? 0 : 1;
}
for (k=0;k<nz;k++) {
ierr = mm_read_mtx_crd_entry(f,&i,&j,&re,&im,type);CHKERRQ(ierr);
i--; j--;
if (i!=j) {
if (i>=low && i<high) {
if (j>=lowj && j<highj)
d_nz[i-low]++;
else
o_nz[i-low]++;
}
if (j>=low && j<high && !mm_is_general(type)) {
if (i>=low && i<high)
d_nz[j-low]++;
else
o_nz[j-low]++;
}
}
}
ierr = preallocation(*M,d_nz,o_nz);CHKERRQ(ierr);
ierr = PetscFree(d_nz);CHKERRQ(ierr);
ierr = PetscFree(o_nz);CHKERRQ(ierr);
/* second read to load the values */
ierr = fseek(f, pos, SEEK_SET);
if (ierr) SETERRQ1(PETSC_COMM_SELF,1,"fseek: %s",strerror(errno));
re = 1.0;
im = 0.0;
/* Set the diagonal to zero */
for (i=low; i<PetscMin(high,n); i++) {
ierr = MatSetValue(*M,i,i,0.0,INSERT_VALUES);CHKERRQ(ierr);
}
for (k=0;k<nz;k++) {
ierr = mm_read_mtx_crd_entry(f,&i,&j,&re,&im,type);
i--; j--;
if (i>=low && i<high) {
s = re + IMAGINARY * im;
ierr = MatSetValue(*M,i,j,s,INSERT_VALUES);CHKERRQ(ierr);
}
if (j>=low && j<high && i != j && !mm_is_general(type)) {
if (mm_is_symmetric(type)) s = re + IMAGINARY * im;
else if (mm_is_hermitian(type)) s = re - IMAGINARY * im;
else if (mm_is_skew(type)) s = -re - IMAGINARY * im;
else {
SETERRQ1(PETSC_COMM_SELF,1,"Matrix format '%s' not supported",mm_typecode_to_str(type));
}
ierr = MatSetValue(*M,j,i,s,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(*M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*M,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (mm_is_symmetric(type)) {
ierr = MatSetOption(*M,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
}
if ((mm_is_symmetric(type) && mm_is_real(type)) || mm_is_hermitian(type)) {
ierr = MatSetOption(*M,MAT_HERMITIAN,PETSC_TRUE);CHKERRQ(ierr);
}
ierr = fclose(f);
if (ierr) SETERRQ1(PETSC_COMM_SELF,1,"fclose: %s",strerror(errno));
PetscFunctionReturn(0);
}
