PetscErrorCode NonlinearGS(SNES snes,Vec X,Vec B,void *ptr)
{
/* values for each basis function at each quadrature point */
AppCtx *user = (AppCtx*)ptr;
PetscInt i,j,k,l,m,n,s;
PetscInt pi,pj,pk;
Field ef[1];
Field ex[8];
PetscScalar ej[9];
CoordField ec[8];
PetscScalar pjac[9],pjinv[9];
PetscScalar pf[3],py[3];
PetscErrorCode ierr;
PetscInt xs,ys,zs;
PetscInt xm,ym,zm;
PetscInt mx,my,mz;
DM cda;
CoordField ***c;
Vec C;
DM da;
Vec Xl,Bl;
Field ***x,***b;
PetscInt sweeps,its;
PetscReal atol,rtol,stol;
PetscReal fnorm0 = 0.0,fnorm,ynorm,xnorm = 0.0;
PetscFunctionBegin;
ierr = SNESNGSGetSweeps(snes,&sweeps);CHKERRQ(ierr);
ierr = SNESNGSGetTolerances(snes,&atol,&rtol,&stol,&its);CHKERRQ(ierr);
ierr = SNESGetDM(snes,&da);CHKERRQ(ierr);
ierr = DMGetLocalVector(da,&Xl);CHKERRQ(ierr);
if (B) {
ierr = DMGetLocalVector(da,&Bl);CHKERRQ(ierr);
}
ierr = DMGlobalToLocalBegin(da,X,INSERT_VALUES,Xl);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,X,INSERT_VALUES,Xl);CHKERRQ(ierr);
if (B) {
ierr = DMGlobalToLocalBegin(da,B,INSERT_VALUES,Bl);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,B,INSERT_VALUES,Bl);CHKERRQ(ierr);
}
ierr = DMDAVecGetArray(da,Xl,&x);CHKERRQ(ierr);
if (B) ierr = DMDAVecGetArray(da,Bl,&b);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da,&cda);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(da,&C);CHKERRQ(ierr);
ierr = DMDAVecGetArray(cda,C,&c);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,&my,&mz,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
for (s=0;s<sweeps;s++) {
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
if (OnBoundary(i,j,k,mx,my,mz)) {
BoundaryValue(i,j,k,mx,my,mz,x[k][j][i],user);
} else {
for (n=0;n<its;n++) {
for (m=0;m<9;m++) pjac[m] = 0.;
for (m=0;m<3;m++) pf[m] = 0.;
/* gather the elements for this point */
for (pk=-1; pk<1; pk++) {
for (pj=-1; pj<1; pj++) {
for (pi=-1; pi<1; pi++) {
/* check that this element exists */
if (i+pi >= 0 && i+pi < mx-1 && j+pj >= 0 && j+pj < my-1 && k+pk >= 0 && k+pk < mz-1) {
/* create the element function and jacobian */
GatherElementData(mx,my,mz,x,c,i+pi,j+pj,k+pk,ex,ec,user);
FormPBJacobian(-pi,-pj,-pk,ex,ec,ef,ej,user);
/* extract the point named by i,j,k from the whole element jacobian and function */
for (l=0;l<3;l++) {
pf[l] += ef[0][l];
for (m=0;m<3;m++) {
pjac[3*m+l] += ej[3*m+l];
}
}
}
}
}
}
/* invert */
InvertTensor(pjac,pjinv,NULL);
/* apply */
if (B) for (m=0;m<3;m++) {
pf[m] -= b[k][j][i][m];
}
TensorVector(pjinv,pf,py);
xnorm=0.;
for (m=0;m<3;m++) {
x[k][j][i][m] -= py[m];
xnorm += PetscRealPart(x[k][j][i][m]*x[k][j][i][m]);
}
fnorm = PetscRealPart(pf[0]*pf[0]+pf[1]*pf[1]+pf[2]*pf[2]);
if (n==0) fnorm0 = fnorm;
ynorm = PetscRealPart(py[0]*py[0]+py[1]*py[1]+py[2]*py[2]);
if (fnorm < atol*atol || fnorm < rtol*rtol*fnorm0 || ynorm < stol*stol*xnorm) break;
}
}
}
}
}
}
ierr = DMDAVecRestoreArray(da,Xl,&x);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,Xl,INSERT_VALUES,X);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,Xl,INSERT_VALUES,X);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(da,&Xl);CHKERRQ(ierr);
if (B) {
ierr = DMDAVecRestoreArray(da,Bl,&b);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(da,&Bl);CHKERRQ(ierr);
}
ierr = DMDAVecRestoreArray(cda,C,&c);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FormJacobianLocal(DMDALocalInfo *info,Field ***x,Mat jacpre,Mat jac,void *ptr)
{
/* values for each basis function at each quadrature point */
AppCtx *user = (AppCtx*)ptr;
PetscInt i,j,k,m,l;
PetscInt ii,jj,kk;
PetscScalar ej[NPB*NPB];
PetscScalar vals[NPB*NPB];
Field ex[NEB];
CoordField ec[NEB];
PetscErrorCode ierr;
PetscInt xs=info->xs,ys=info->ys,zs=info->zs;
PetscInt xm=info->xm,ym=info->ym,zm=info->zm;
PetscInt xes,yes,zes,xee,yee,zee;
PetscInt mx=info->mx,my=info->my,mz=info->mz;
DM cda;
CoordField ***c;
Vec C;
PetscInt nrows;
MatStencil col[NPB],row[NPB];
PetscScalar v[9];
PetscFunctionBegin;
ierr = DMGetCoordinateDM(info->da,&cda);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(info->da,&C);CHKERRQ(ierr);
ierr = DMDAVecGetArray(cda,C,&c);CHKERRQ(ierr);
ierr = MatScale(jac,0.0);CHKERRQ(ierr);
xes = xs;
yes = ys;
zes = zs;
xee = xs+xm;
yee = ys+ym;
zee = zs+zm;
if (xs > 0) xes = xs-1;
if (ys > 0) yes = ys-1;
if (zs > 0) zes = zs-1;
if (xs+xm == mx) xee = xs+xm-1;
if (ys+ym == my) yee = ys+ym-1;
if (zs+zm == mz) zee = zs+zm-1;
for (k=zes; k<zee; k++) {
for (j=yes; j<yee; j++) {
for (i=xes; i<xee; i++) {
GatherElementData(mx,my,mz,x,c,i,j,k,ex,ec,user);
FormElementJacobian(ex,ec,NULL,ej,user);
ApplyBCsElement(mx,my,mz,i,j,k,ej);
nrows = 0.;
for (kk=0;kk<NB;kk++){
for (jj=0;jj<NB;jj++) {
for (ii=0;ii<NB;ii++) {
PetscInt idx = ii + jj*2 + kk*4;
for (m=0;m<3;m++) {
col[3*idx+m].i = i+ii;
col[3*idx+m].j = j+jj;
col[3*idx+m].k = k+kk;
col[3*idx+m].c = m;
if (i+ii >= xs && i+ii < xm+xs && j+jj >= ys && j+jj < ys+ym && k+kk >= zs && k+kk < zs+zm) {
row[nrows].i = i+ii;
row[nrows].j = j+jj;
row[nrows].k = k+kk;
row[nrows].c = m;
for (l=0;l<NPB;l++) vals[NPB*nrows + l] = ej[NPB*(3*idx+m) + l];
nrows++;
}
}
}
}
}
ierr = MatSetValuesStencil(jac,nrows,row,NPB,col,vals,ADD_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FLUSH_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FLUSH_ASSEMBLY);CHKERRQ(ierr);
/* set the diagonal */
v[0] = 1.;v[1] = 0.;v[2] = 0.;v[3] = 0.;v[4] = 1.;v[5] = 0.;v[6] = 0.;v[7] = 0.;v[8] = 1.;
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
if (OnBoundary(i,j,k,mx,my,mz)) {
for (m=0; m<3;m++) {
col[m].i = i;
col[m].j = j;
col[m].k = k;
col[m].c = m;
}
ierr = MatSetValuesStencil(jac,3,col,3,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = DMDAVecRestoreArray(cda,C,&c);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FormFunctionLocal(DMDALocalInfo *info,Field ***x,Field ***f,void *ptr)
{
/* values for each basis function at each quadrature point */
AppCtx *user = (AppCtx*)ptr;
PetscInt i,j,k,l;
PetscInt ii,jj,kk;
Field ef[NEB];
Field ex[NEB];
CoordField ec[NEB];
PetscErrorCode ierr;
PetscInt xs=info->xs,ys=info->ys,zs=info->zs;
PetscInt xm=info->xm,ym=info->ym,zm=info->zm;
PetscInt xes,yes,zes,xee,yee,zee;
PetscInt mx=info->mx,my=info->my,mz=info->mz;
DM cda;
CoordField ***c;
Vec C;
PetscFunctionBegin;
ierr = DMGetCoordinateDM(info->da,&cda);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(info->da,&C);CHKERRQ(ierr);
ierr = DMDAVecGetArray(cda,C,&c);CHKERRQ(ierr);
ierr = DMDAGetInfo(info->da,0,&mx,&my,&mz,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetCorners(info->da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
/* loop over elements */
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
for (l=0;l<3;l++) {
f[k][j][i][l] = 0.;
}
}
}
}
/* element starts and ends */
xes = xs;
yes = ys;
zes = zs;
xee = xs+xm;
yee = ys+ym;
zee = zs+zm;
if (xs > 0) xes = xs - 1;
if (ys > 0) yes = ys - 1;
if (zs > 0) zes = zs - 1;
if (xs+xm == mx) xee = xs+xm-1;
if (ys+ym == my) yee = ys+ym-1;
if (zs+zm == mz) zee = zs+zm-1;
for (k=zes; k<zee; k++) {
for (j=yes; j<yee; j++) {
for (i=xes; i<xee; i++) {
GatherElementData(mx,my,mz,x,c,i,j,k,ex,ec,user);
FormElementJacobian(ex,ec,ef,NULL,user);
/* put this element's additions into the residuals */
for (kk=0;kk<NB;kk++){
for (jj=0;jj<NB;jj++) {
for (ii=0;ii<NB;ii++) {
PetscInt idx = ii + jj*NB + kk*NB*NB;
if (k+kk >= zs && j+jj >= ys && i+ii >= xs && k+kk < zs+zm && j+jj < ys+ym && i+ii < xs+xm) {
if (OnBoundary(i+ii,j+jj,k+kk,mx,my,mz)) {
for (l=0;l<3;l++)
f[k+kk][j+jj][i+ii][l] = x[k+kk][j+jj][i+ii][l] - ex[idx][l];
} else {
for (l=0;l<3;l++)
f[k+kk][j+jj][i+ii][l] += ef[idx][l];
}
}
}
}
}
}
}
}
ierr = DMDAVecRestoreArray(cda,C,&c);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMCoarsen_Plex(DM dm, MPI_Comm comm, DM *dmCoarsened)
{
DM_Plex *mesh = (DM_Plex*) dm->data;
#ifdef PETSC_HAVE_PRAGMATIC
DM udm, coordDM;
DMLabel bd;
Mat A;
Vec coordinates, mb, mx;
PetscSection coordSection;
const PetscScalar *coords;
double *coarseCoords;
IS bdIS;
PetscReal *x, *y, *z, *eqns, *metric;
PetscReal coarseRatio = PetscSqr(0.5);
const PetscInt *faces;
PetscInt *cells, *bdFaces, *bdFaceIds;
PetscInt dim, numCorners, cStart, cEnd, numCells, numCoarseCells, c, vStart, vEnd, numVertices, numCoarseVertices, v, numBdFaces, f, maxConeSize, size, bdSize, coff;
#endif
PetscErrorCode ierr;
PetscFunctionBegin;
#ifdef PETSC_HAVE_PRAGMATIC
if (!mesh->coarseMesh) {
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm, &coordDM);CHKERRQ(ierr);
ierr = DMGetDefaultSection(coordDM, &coordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexUninterpolate(dm, &udm);CHKERRQ(ierr);
ierr = DMPlexGetMaxSizes(udm, &maxConeSize, NULL);CHKERRQ(ierr);
numCells = cEnd - cStart;
numVertices = vEnd - vStart;
ierr = PetscCalloc5(numVertices, &x, numVertices, &y, numVertices, &z, numVertices*PetscSqr(dim), &metric, numCells*maxConeSize, &cells);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (v = vStart; v < vEnd; ++v) {
PetscInt off;
ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
x[v-vStart] = coords[off+0];
y[v-vStart] = coords[off+1];
if (dim > 2) z[v-vStart] = coords[off+2];
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (c = 0, coff = 0; c < numCells; ++c) {
const PetscInt *cone;
PetscInt coneSize, cl;
ierr = DMPlexGetConeSize(udm, c, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(udm, c, &cone);CHKERRQ(ierr);
for (cl = 0; cl < coneSize; ++cl) cells[coff++] = cone[cl] - vStart;
}
switch (dim) {
case 2:
pragmatic_2d_init(&numVertices, &numCells, cells, x, y);
break;
case 3:
pragmatic_3d_init(&numVertices, &numCells, cells, x, y, z);
break;
default:
SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No Pragmatic coarsening defined for dimension %d", dim);
}
/* Create boundary mesh */
ierr = DMLabelCreate("boundary", &bd);CHKERRQ(ierr);
ierr = DMPlexMarkBoundaryFaces(dm, bd);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(bd, 1, &bdIS);CHKERRQ(ierr);
ierr = DMLabelGetStratumSize(bd, 1, &numBdFaces);CHKERRQ(ierr);
ierr = ISGetIndices(bdIS, &faces);CHKERRQ(ierr);
for (f = 0, bdSize = 0; f < numBdFaces; ++f) {
PetscInt *closure = NULL;
PetscInt closureSize, cl;
ierr = DMPlexGetTransitiveClosure(dm, faces[f], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) ++bdSize;
}
ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
ierr = PetscMalloc2(bdSize, &bdFaces, numBdFaces, &bdFaceIds);CHKERRQ(ierr);
for (f = 0, bdSize = 0; f < numBdFaces; ++f) {
PetscInt *closure = NULL;
PetscInt closureSize, cl;
ierr = DMPlexGetTransitiveClosure(dm, faces[f], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (cl = 0; cl < closureSize*2; cl += 2) {
if ((closure[cl] >= vStart) && (closure[cl] < vEnd)) bdFaces[bdSize++] = closure[cl] - vStart;
}
/* TODO Fix */
bdFaceIds[f] = 1;
ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
}
ierr = ISDestroy(&bdIS);CHKERRQ(ierr);
ierr = DMLabelDestroy(&bd);CHKERRQ(ierr);
pragmatic_set_boundary(&numBdFaces, bdFaces, bdFaceIds);
/* Create metric */
size = (dim*(dim+1))/2;
ierr = PetscMalloc1(PetscSqr(size), &eqns);CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF, size, size, eqns, &A);CHKERRQ(ierr);
ierr = MatCreateVecs(A, &mx, &mb);CHKERRQ(ierr);
ierr = VecSet(mb, 1.0);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
const PetscScalar *sol;
PetscScalar *cellCoords = NULL;
PetscReal e[3], vol;
const PetscInt *cone;
PetscInt coneSize, cl, i, j, d, r;
ierr = DMPlexVecGetClosure(dm, coordSection, coordinates, c, NULL, &cellCoords);CHKERRQ(ierr);
/* Only works for simplices */
for (i = 0, r = 0; i < dim+1; ++i) {
for (j = 0; j < i; ++j, ++r) {
for (d = 0; d < dim; ++d) e[d] = cellCoords[i*dim+d] - cellCoords[j*dim+d];
/* FORTRAN ORDERING */
if (dim == 2) {
eqns[0*size+r] = PetscSqr(e[0]);
eqns[1*size+r] = 2.0*e[0]*e[1];
eqns[2*size+r] = PetscSqr(e[1]);
} else {
eqns[0*size+r] = PetscSqr(e[0]);
eqns[1*size+r] = 2.0*e[0]*e[1];
eqns[2*size+r] = 2.0*e[0]*e[2];
eqns[3*size+r] = PetscSqr(e[1]);
eqns[4*size+r] = 2.0*e[1]*e[2];
eqns[5*size+r] = PetscSqr(e[2]);
}
}
}
ierr = MatSetUnfactored(A);CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dm, coordSection, coordinates, c, NULL, &cellCoords);CHKERRQ(ierr);
ierr = MatLUFactor(A, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = MatSolve(A, mb, mx);CHKERRQ(ierr);
ierr = VecGetArrayRead(mx, &sol);CHKERRQ(ierr);
ierr = DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL);CHKERRQ(ierr);
ierr = DMPlexGetCone(udm, c, &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(udm, c, &coneSize);CHKERRQ(ierr);
for (cl = 0; cl < coneSize; ++cl) {
const PetscInt v = cone[cl] - vStart;
if (dim == 2) {
metric[v*4+0] += vol*coarseRatio*sol[0];
metric[v*4+1] += vol*coarseRatio*sol[1];
metric[v*4+2] += vol*coarseRatio*sol[1];
metric[v*4+3] += vol*coarseRatio*sol[2];
} else {
metric[v*9+0] += vol*coarseRatio*sol[0];
metric[v*9+1] += vol*coarseRatio*sol[1];
metric[v*9+3] += vol*coarseRatio*sol[1];
metric[v*9+2] += vol*coarseRatio*sol[2];
metric[v*9+6] += vol*coarseRatio*sol[2];
metric[v*9+4] += vol*coarseRatio*sol[3];
metric[v*9+5] += vol*coarseRatio*sol[4];
metric[v*9+7] += vol*coarseRatio*sol[4];
metric[v*9+8] += vol*coarseRatio*sol[5];
}
}
ierr = VecRestoreArrayRead(mx, &sol);CHKERRQ(ierr);
}
for (v = 0; v < numVertices; ++v) {
const PetscInt *support;
PetscInt supportSize, s;
PetscReal vol, totVol = 0.0;
ierr = DMPlexGetSupport(udm, v+vStart, &support);CHKERRQ(ierr);
ierr = DMPlexGetSupportSize(udm, v+vStart, &supportSize);CHKERRQ(ierr);
for (s = 0; s < supportSize; ++s) {ierr = DMPlexComputeCellGeometryFVM(dm, support[s], &vol, NULL, NULL);CHKERRQ(ierr); totVol += vol;}
for (s = 0; s < PetscSqr(dim); ++s) metric[v*PetscSqr(dim)+s] /= totVol;
}
ierr = VecDestroy(&mx);CHKERRQ(ierr);
ierr = VecDestroy(&mb);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = DMDestroy(&udm);CHKERRQ(ierr);
ierr = PetscFree(eqns);CHKERRQ(ierr);
pragmatic_set_metric(metric);
pragmatic_adapt();
/* Read out mesh */
pragmatic_get_info(&numCoarseVertices, &numCoarseCells);
ierr = PetscMalloc1(numCoarseVertices*dim, &coarseCoords);CHKERRQ(ierr);
switch (dim) {
case 2:
pragmatic_get_coords_2d(x, y);
numCorners = 3;
for (v = 0; v < numCoarseVertices; ++v) {coarseCoords[v*2+0] = x[v]; coarseCoords[v*2+1] = y[v];}
break;
case 3:
pragmatic_get_coords_3d(x, y, z);
numCorners = 4;
for (v = 0; v < numCoarseVertices; ++v) {coarseCoords[v*3+0] = x[v]; coarseCoords[v*3+1] = y[v]; coarseCoords[v*3+2] = z[v];}
break;
default:
SETERRQ1(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "No Pragmatic coarsening defined for dimension %d", dim);
}
pragmatic_get_elements(cells);
/* TODO Read out markers for boundary */
ierr = DMPlexCreateFromCellList(PetscObjectComm((PetscObject) dm), dim, numCoarseCells, numCoarseVertices, numCorners, PETSC_TRUE, cells, dim, coarseCoords, &mesh->coarseMesh);CHKERRQ(ierr);
pragmatic_finalize();
ierr = PetscFree5(x, y, z, metric, cells);CHKERRQ(ierr);
ierr = PetscFree2(bdFaces, bdFaceIds);CHKERRQ(ierr);
ierr = PetscFree(coarseCoords);CHKERRQ(ierr);
}
#endif
ierr = PetscObjectReference((PetscObject) mesh->coarseMesh);CHKERRQ(ierr);
*dmCoarsened = mesh->coarseMesh;
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
DM dm;
PetscBool interpolate = PETSC_TRUE;
PetscInt dim;
PetscInt pStart, pEnd;
PetscInt vStart, vEnd;
PetscInt eStart, eEnd;
PetscInt cStart, cEnd;
PetscSection s;
ierr = PetscInitialize(&argc, &argv, (char*)0, help); CHKERRQ(ierr);
ierr = DMPlexCreateGmshFromFile(MPI_COMM_WORLD, argv[1], interpolate, &dm); CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim); CHKERRQ(ierr);
printf("dim = %d\n", dim);
ierr = DMPlexGetChart(dm, &pStart, &pEnd); CHKERRQ(ierr);
printf("chart: pstart, pend = %d %d\n", pStart, pEnd);
// vertices
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd); CHKERRQ(ierr);
printf("Depth 0: vstart, vend, len = %d %d %d\n", vStart, vEnd, vEnd-vStart);
// edges
ierr = DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd); CHKERRQ(ierr);
printf("Depth 1: estart, eend, len = %d %d %d\n", eStart, eEnd, eEnd-eStart);
// cells
ierr = DMPlexGetDepthStratum(dm, 2, &cStart, &cEnd); CHKERRQ(ierr);
printf("Depth 2: cstart, cend, len = %d %d %d\n", cStart, cEnd, cEnd-cStart);
// cells adjacent to face
{
FILE * fid = fopen("face_nbr.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
PetscInt nbr;
ierr = DMPlexGetSupportSize(dm, e, &nbr); CHKERRQ(ierr);
const PetscInt *nbcells;
ierr = DMPlexGetSupport(dm, e, &nbcells); CHKERRQ(ierr);
if(nbr == 1) // boundary face
fprintf(fid, "%d %d\n",e-eStart+1,nbcells[0]-cStart+1);
else if(nbr == 2) // interior face
fprintf(fid, "%d %d %d\n",e-eStart+1,nbcells[0]-cStart+1,nbcells[1]-cStart+1);
else
{
printf("nbr is not 1 or 2\n");
exit(0);
}
}
fclose(fid);
}
// point coordinates
{
Vec coordinates;
ierr = DMGetCoordinatesLocal(dm, &coordinates); CHKERRQ(ierr);
const PetscScalar *coords;
ierr = VecGetArrayRead(coordinates, &coords); CHKERRQ(ierr);
DM dmCoord;
ierr = DMGetCoordinateDM(dm, &dmCoord); CHKERRQ(ierr);
FILE * fid = fopen("vertices.txt","w");
for(PetscInt v=vStart; v<vEnd; ++v)
{
PetscScalar *vertex;
ierr = DMPlexPointLocalRead(dmCoord, v, coords, &vertex); CHKERRQ(ierr);
fprintf(fid, "%f %f\n", vertex[0], vertex[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(coordinates, &coords); CHKERRQ(ierr);
}
// compute cell and face geometry
{
Vec cellgeom, facegeom;
ierr = DMPlexComputeGeometryFVM(dm, &cellgeom, &facegeom); CHKERRQ(ierr);
// cell information
DM dmCell;
ierr = VecGetDM(cellgeom, &dmCell); CHKERRQ(ierr);
const PetscScalar *cgeom;
ierr = VecGetArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
FILE * fid = fopen("cells.txt","w");
for(PetscInt c=cStart; c<cEnd; ++c)
{
// cell properties like volume, centroid
PetscFVCellGeom *cg;
ierr = DMPlexPointLocalRead(dmCell, c, cgeom, &cg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f\n", c-cStart+1, cg->volume,
cg->centroid[0], cg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(cellgeom, &cgeom); CHKERRQ(ierr);
// face information
DM dmFace;
ierr = VecGetDM(facegeom, &dmFace); CHKERRQ(ierr);
const PetscScalar *fgeom;
ierr = VecGetArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
fid = fopen("faces.txt","w");
for(PetscInt e=eStart; e<eEnd; ++e)
{
// face properties like area normal, centroid
PetscFVFaceGeom *fg;
ierr = DMPlexPointLocalRead(dmFace, e, fgeom, &fg); CHKERRQ(ierr);
fprintf(fid, "%d %f %f %f %f\n", e-eStart+1, fg->normal[0], fg->normal[1],
fg->centroid[0], fg->centroid[1]);
}
fclose(fid);
ierr = VecRestoreArrayRead(facegeom, &fgeom); CHKERRQ(ierr);
}
// create section with one variable in each cell
ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &s); CHKERRQ(ierr);
ierr = PetscSectionSetChart(s, pStart, pEnd); CHKERRQ(ierr);
for(PetscInt c=cStart; c<cEnd; ++c)
{
ierr = PetscSectionSetDof(s, c, 1); CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(s); CHKERRQ(ierr);
// create vector to store solution
Vec lv, gv;
ierr = DMSetDefaultSection(dm, s); CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &lv); CHKERRQ(ierr);
ierr = DMGetGlobalVector(dm, &gv); CHKERRQ(ierr);
}
static PetscErrorCode ComputeSpectral(DM dm, Vec u, PetscInt numPlanes, const PetscInt planeDir[], const PetscReal planeCoord[], AppCtx *user)
{
MPI_Comm comm;
PetscSection coordSection, section;
Vec coordinates, uloc;
const PetscScalar *coords, *array;
PetscInt p;
PetscMPIInt size, rank;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm, u, INSERT_VALUES, uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm, u, INSERT_VALUES, uloc);CHKERRQ(ierr);
ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, uloc, 0.0, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = VecViewFromOptions(uloc, NULL, "-sol_view");CHKERRQ(ierr);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = VecGetArrayRead(uloc, &array);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (p = 0; p < numPlanes; ++p) {
DMLabel label;
char name[PETSC_MAX_PATH_LEN];
Mat F;
Vec x, y;
IS stratum;
PetscReal *ray, *gray;
PetscScalar *rvals, *svals, *gsvals;
PetscInt *perm, *nperm;
PetscInt n, N, i, j, off, offu;
const PetscInt *points;
ierr = PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "spectral_plane_%D", p);CHKERRQ(ierr);
ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(label, 1, &stratum);CHKERRQ(ierr);
ierr = ISGetLocalSize(stratum, &n);CHKERRQ(ierr);
ierr = ISGetIndices(stratum, &points);CHKERRQ(ierr);
ierr = PetscMalloc2(n, &ray, n, &svals);CHKERRQ(ierr);
for (i = 0; i < n; ++i) {
ierr = PetscSectionGetOffset(coordSection, points[i], &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, points[i], &offu);CHKERRQ(ierr);
ray[i] = PetscRealPart(coords[off+((planeDir[p]+1)%2)]);
svals[i] = array[offu];
}
/* Gather the ray data to proc 0 */
if (size > 1) {
PetscMPIInt *cnt, *displs, p;
ierr = PetscCalloc2(size, &cnt, size, &displs);CHKERRQ(ierr);
ierr = MPI_Gather(&n, 1, MPIU_INT, cnt, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 1; p < size; ++p) displs[p] = displs[p-1] + cnt[p-1];
N = displs[size-1] + cnt[size-1];
ierr = PetscMalloc2(N, &gray, N, &gsvals);CHKERRQ(ierr);
ierr = MPI_Gatherv(ray, n, MPIU_REAL, gray, cnt, displs, MPIU_REAL, 0, comm);CHKERRQ(ierr);
ierr = MPI_Gatherv(svals, n, MPIU_SCALAR, gsvals, cnt, displs, MPIU_SCALAR, 0, comm);CHKERRQ(ierr);
ierr = PetscFree2(cnt, displs);CHKERRQ(ierr);
} else {
N = n;
gray = ray;
gsvals = svals;
}
if (!rank) {
/* Sort point along ray */
ierr = PetscMalloc2(N, &perm, N, &nperm);CHKERRQ(ierr);
for (i = 0; i < N; ++i) {perm[i] = i;}
ierr = PetscSortRealWithPermutation(N, gray, perm);CHKERRQ(ierr);
/* Count duplicates and squish mapping */
nperm[0] = perm[0];
for (i = 1, j = 1; i < N; ++i) {
if (PetscAbsReal(gray[perm[i]] - gray[perm[i-1]]) > PETSC_SMALL) nperm[j++] = perm[i];
}
/* Create FFT structs */
ierr = MatCreateFFT(PETSC_COMM_SELF, 1, &j, MATFFTW, &F);CHKERRQ(ierr);
ierr = MatCreateVecs(F, &x, &y);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, name);CHKERRQ(ierr);
ierr = VecGetArray(x, &rvals);CHKERRQ(ierr);
for (i = 0, j = 0; i < N; ++i) {
if (i > 0 && PetscAbsReal(gray[perm[i]] - gray[perm[i-1]]) < PETSC_SMALL) continue;
rvals[j] = gsvals[nperm[j]];
++j;
}
ierr = PetscFree2(perm, nperm);CHKERRQ(ierr);
if (size > 1) {ierr = PetscFree2(gray, gsvals);CHKERRQ(ierr);}
ierr = VecRestoreArray(x, &rvals);CHKERRQ(ierr);
/* Do FFT along the ray */
ierr = MatMult(F, x, y);CHKERRQ(ierr);
/* Chop FFT */
ierr = VecChop(y, PETSC_SMALL);CHKERRQ(ierr);
ierr = VecViewFromOptions(x, NULL, "-real_view");CHKERRQ(ierr);
ierr = VecViewFromOptions(y, NULL, "-fft_view");CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(stratum, &points);CHKERRQ(ierr);
ierr = ISDestroy(&stratum);CHKERRQ(ierr);
ierr = PetscFree2(ray, svals);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(uloc, &array);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &uloc);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode OpGetDiagonal(Op op,DM dm,Vec Diag) {
PetscErrorCode ierr;
Vec X,Vl;
PetscInt nelem,P,Q,P3,Q3,NE;
DM dmx;
const PetscScalar *x;
PetscScalar *diag;
const PetscReal *B,*D,*w3;
PetscFunctionBegin;
if (!op->PointwiseElement) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_USER,"No PointwiseElement implemented, use OpSetPointwiseElement()");
if (op->dof != 1) SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"dof != 1");
ierr = PetscLogEventBegin(OP_GetDiagonal,dm,Diag,0,0);CHKERRQ(ierr);
ierr = DMFEGetTensorEval(dm,&P,&Q,&B,&D,NULL,NULL,&w3);CHKERRQ(ierr);
P3 = P*P*P;
Q3 = Q*Q*Q;
NE = op->ne;
ierr = DMGetLocalVector(dm,&Vl);CHKERRQ(ierr);
ierr = VecZeroEntries(Vl);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm,&dmx);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm,&X);CHKERRQ(ierr);
ierr = DMFEGetNumElements(dm,&nelem);CHKERRQ(ierr);
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
ierr = VecGetArray(Vl,&diag);CHKERRQ(ierr);
for (PetscInt e=0; e<nelem; e+=NE) {
PetscScalar diage[1*P3*NE]_align,ve[1*P3*NE]_align,dv[3][1][Q3][NE]_align,ue[1*P3*NE]_align,du[3][1][Q3][NE]_align,xe[3*P3*NE]_align,dx[3][3][Q3][NE]_align,wdxdet[Q3][NE]_align;
ierr = DMFEExtractElements(dmx,x,e,NE,xe);CHKERRQ(ierr);
ierr = PetscMemzero(dx,sizeof dx);CHKERRQ(ierr);
ierr = TensorContract(op->Tensor3,D,B,B,TENSOR_EVAL,xe,dx[0][0][0]);CHKERRQ(ierr);
ierr = TensorContract(op->Tensor3,B,D,B,TENSOR_EVAL,xe,dx[1][0][0]);CHKERRQ(ierr);
ierr = TensorContract(op->Tensor3,B,B,D,TENSOR_EVAL,xe,dx[2][0][0]);CHKERRQ(ierr);
ierr = PointwiseJacobianInvert(NE,Q*Q*Q,w3,dx,wdxdet);CHKERRQ(ierr);
for (PetscInt i=0; i<P3; i++) {
ierr = PetscMemzero(ue,sizeof ue);CHKERRQ(ierr);
for (PetscInt k=0; k<op->ne; k++) ue[i*NE+k] = 1;
ierr = PetscMemzero(du,sizeof du);CHKERRQ(ierr);
ierr = TensorContract(op->TensorDOF,D,B,B,TENSOR_EVAL,ue,du[0][0][0]);CHKERRQ(ierr);
ierr = TensorContract(op->TensorDOF,B,D,B,TENSOR_EVAL,ue,du[1][0][0]);CHKERRQ(ierr);
ierr = TensorContract(op->TensorDOF,B,B,D,TENSOR_EVAL,ue,du[2][0][0]);CHKERRQ(ierr);
ierr = (*op->PointwiseElement)(op,NE,Q3,dx[0][0][0],wdxdet[0],du[0][0][0],dv[0][0][0]);CHKERRQ(ierr);
ierr = PetscMemzero(ve,sizeof ve);CHKERRQ(ierr);
ierr = TensorContract(op->TensorDOF,D,B,B,TENSOR_TRANSPOSE,dv[0][0][0],ve);CHKERRQ(ierr);
ierr = TensorContract(op->TensorDOF,B,D,B,TENSOR_TRANSPOSE,dv[1][0][0],ve);CHKERRQ(ierr);
ierr = TensorContract(op->TensorDOF,B,B,D,TENSOR_TRANSPOSE,dv[2][0][0],ve);CHKERRQ(ierr);
for (PetscInt k=0; k<op->ne; k++) diage[i*NE+k] = ve[i*NE+k];
}
ierr = DMFESetElements(dm,diag,e,NE,ADD_VALUES,DOMAIN_INTERIOR,diage);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
ierr = VecRestoreArray(Vl,&diag);CHKERRQ(ierr);
ierr = VecZeroEntries(Diag);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(dm,Vl,ADD_VALUES,Diag);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(dm,Vl,ADD_VALUES,Diag);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm,&Vl);CHKERRQ(ierr);
ierr = PetscLogEventEnd(OP_GetDiagonal,dm,Diag,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
// Approximates continuous L^\infty and L^2 norms of error, normalized by L^\infty and L^2 norms of analytic solution
PetscErrorCode OpIntegrateNorms(Op op,DM dm,Vec U,PetscReal *normInfty,PetscReal *normL2) {
PetscErrorCode ierr;
Vec X,Uloc;
DM dmx;
const PetscScalar *x,*u;
const PetscReal *B,*D,*w3;
PetscReal L[3];
struct {PetscReal error,u;} sumInfty={},sum2={};
PetscInt nelem,ne = op->ne,P,Q,P3,Q3;
PetscFunctionBegin;
ierr = PetscLogEventBegin(OP_IntegrateNorms,dm,U,0,0);CHKERRQ(ierr);
ierr = DMFEGetTensorEval(dm,&P,&Q,&B,&D,NULL,NULL,&w3);CHKERRQ(ierr);
P3 = P*P*P;
Q3 = Q*Q*Q;
ierr = DMFEGetUniformCoordinates(dm,L);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm,&dmx);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm,&X);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm,&Uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm,U,INSERT_VALUES,Uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm,U,INSERT_VALUES,Uloc);CHKERRQ(ierr);
ierr = DMFEGetNumElements(dm,&nelem);CHKERRQ(ierr);
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
ierr = VecGetArrayRead(Uloc,&u);CHKERRQ(ierr);
for (PetscInt e=0; e<nelem; e+=ne) {
PetscScalar ue[op->dof*P3*ne]_align,uq[op->dof][Q3][ne]_align,xe[3*P3*ne]_align,xq[3][Q3][ne]_align,dx[3][3][Q3][ne]_align,wdxdet[Q3][ne]_align;
ierr = DMFEExtractElements(dmx,x,e,ne,xe);CHKERRQ(ierr);
ierr = PetscMemzero(xq,sizeof xq);CHKERRQ(ierr);
ierr = TensorContract(op->Tensor3,B,B,B,TENSOR_EVAL,xe,xq[0][0]);CHKERRQ(ierr);
ierr = PetscMemzero(dx,sizeof dx);CHKERRQ(ierr);
ierr = TensorContract(op->Tensor3,D,B,B,TENSOR_EVAL,xe,dx[0][0][0]);CHKERRQ(ierr);
ierr = TensorContract(op->Tensor3,B,D,B,TENSOR_EVAL,xe,dx[1][0][0]);CHKERRQ(ierr);
ierr = TensorContract(op->Tensor3,B,B,D,TENSOR_EVAL,xe,dx[2][0][0]);CHKERRQ(ierr);
ierr = PointwiseJacobianInvert(ne,Q3,w3,dx,wdxdet);CHKERRQ(ierr);
ierr = DMFEExtractElements(dm,u,e,ne,ue);CHKERRQ(ierr);
ierr = PetscMemzero(uq,sizeof uq);CHKERRQ(ierr);
ierr = TensorContract(op->TensorDOF,B,B,B,TENSOR_EVAL,ue,uq[0][0]);CHKERRQ(ierr);
for (PetscInt i=0; i<Q3; i++) {
for (PetscInt l=0; l<ne; l++) {
PetscReal xx[] = {xq[0][i][l],xq[1][i][l],xq[2][i][l]};
PetscScalar uql[op->dof],fql[op->dof];
ierr = (op->PointwiseSolution)(op,xx,L,uql);CHKERRQ(ierr);
ierr = (op->PointwiseForcing)(op,xx,L,fql);CHKERRQ(ierr);
for (PetscInt d=0; d<op->dof; d++) {
PetscReal error = uq[d][i][l] - uql[d];
sumInfty.error = PetscMax(sumInfty.error,PetscAbs(error));
sumInfty.u = PetscMax(sumInfty.u ,PetscAbs(uql[d]));
sum2.error += PetscSqr(error) * wdxdet[i][l];
sum2.u += PetscSqr(uql[d]) * wdxdet[i][l];
}
}
}
}
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(Uloc,&u);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm,&Uloc);CHKERRQ(ierr);
ierr = MPI_Allreduce(MPI_IN_PLACE,(void*)&sumInfty,2,MPIU_REAL,MPIU_MAX,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
ierr = MPI_Allreduce(MPI_IN_PLACE,(void*)&sum2,2,MPIU_REAL,MPIU_SUM,PetscObjectComm((PetscObject)dm));CHKERRQ(ierr);
*normInfty = sumInfty.error/sumInfty.u;
*normL2 = PetscSqrtReal(sum2.error)/PetscSqrtReal(sum2.u);
ierr = PetscLogEventEnd(OP_IntegrateNorms,dm,U,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_STATIC_INLINE PetscErrorCode DMInterpolate_Hex_Private(DMInterpolationInfo ctx, DM dm, Vec xLocal, Vec v)
{
DM dmCoord;
SNES snes;
KSP ksp;
PC pc;
Vec coordsLocal, r, ref, real;
Mat J;
PetscScalar *a, *coords;
PetscInt p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm, &dmCoord);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_SELF, &snes);CHKERRQ(ierr);
ierr = SNESSetOptionsPrefix(snes, "hex_interp_");CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &r);CHKERRQ(ierr);
ierr = VecSetSizes(r, 3, 3);CHKERRQ(ierr);
ierr = VecSetType(r,dm->vectype);CHKERRQ(ierr);
ierr = VecDuplicate(r, &ref);CHKERRQ(ierr);
ierr = VecDuplicate(r, &real);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_SELF, &J);CHKERRQ(ierr);
ierr = MatSetSizes(J, 3, 3, 3, 3);CHKERRQ(ierr);
ierr = MatSetType(J, MATSEQDENSE);CHKERRQ(ierr);
ierr = MatSetUp(J);CHKERRQ(ierr);
ierr = SNESSetFunction(snes, r, HexMap_Private, NULL);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, J, J, HexJacobian_Private, NULL);CHKERRQ(ierr);
ierr = SNESGetKSP(snes, &ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr);
ierr = PCSetType(pc, PCLU);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = VecGetArray(ctx->coords, &coords);CHKERRQ(ierr);
ierr = VecGetArray(v, &a);CHKERRQ(ierr);
for (p = 0; p < ctx->n; ++p) {
PetscScalar *x = NULL, *vertices = NULL;
PetscScalar *xi;
PetscReal xir[3];
PetscInt c = ctx->cells[p], comp, coordSize, xSize;
/* Can make this do all points at once */
ierr = DMPlexVecGetClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);CHKERRQ(ierr);
if (8*3 != coordSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", coordSize, 8*3);
ierr = DMPlexVecGetClosure(dm, NULL, xLocal, c, &xSize, &x);CHKERRQ(ierr);
if (8*ctx->dof != xSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", xSize, 8*ctx->dof);
ierr = SNESSetFunction(snes, NULL, NULL, (void*) vertices);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, NULL, NULL, NULL, (void*) vertices);CHKERRQ(ierr);
ierr = VecGetArray(real, &xi);CHKERRQ(ierr);
xi[0] = coords[p*ctx->dim+0];
xi[1] = coords[p*ctx->dim+1];
xi[2] = coords[p*ctx->dim+2];
ierr = VecRestoreArray(real, &xi);CHKERRQ(ierr);
ierr = SNESSolve(snes, real, ref);CHKERRQ(ierr);
ierr = VecGetArray(ref, &xi);CHKERRQ(ierr);
xir[0] = PetscRealPart(xi[0]);
xir[1] = PetscRealPart(xi[1]);
xir[2] = PetscRealPart(xi[2]);
for (comp = 0; comp < ctx->dof; ++comp) {
a[p*ctx->dof+comp] =
x[0*ctx->dof+comp]*(1-xir[0])*(1-xir[1])*(1-xir[2]) +
x[3*ctx->dof+comp]* xir[0]*(1-xir[1])*(1-xir[2]) +
x[2*ctx->dof+comp]* xir[0]* xir[1]*(1-xir[2]) +
x[1*ctx->dof+comp]*(1-xir[0])* xir[1]*(1-xir[2]) +
x[4*ctx->dof+comp]*(1-xir[0])*(1-xir[1])* xir[2] +
x[5*ctx->dof+comp]* xir[0]*(1-xir[1])* xir[2] +
x[6*ctx->dof+comp]* xir[0]* xir[1]* xir[2] +
x[7*ctx->dof+comp]*(1-xir[0])* xir[1]* xir[2];
}
ierr = VecRestoreArray(ref, &xi);CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dm, NULL, xLocal, c, &xSize, &x);CHKERRQ(ierr);
}
ierr = VecRestoreArray(v, &a);CHKERRQ(ierr);
ierr = VecRestoreArray(ctx->coords, &coords);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = VecDestroy(&r);CHKERRQ(ierr);
ierr = VecDestroy(&ref);CHKERRQ(ierr);
ierr = VecDestroy(&real);CHKERRQ(ierr);
ierr = MatDestroy(&J);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Write all fields that have been provided to the viewer
Multi-block XML format with binary appended data.
*/
PetscErrorCode DMPlexVTKWriteAll_VTU(DM dm,PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*)viewer->data;
PetscViewerVTKObjectLink link;
FILE *fp;
PetscMPIInt rank,size,tag;
PetscErrorCode ierr;
PetscInt dim,cellHeight,cStart,cEnd,vStart,vEnd,cMax,numLabelCells,hasLabel,c,v,r,i;
PieceInfo piece,*gpiece = NULL;
void *buffer = NULL;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
#if defined(PETSC_USE_COMPLEX)
SETERRQ(PetscObjectComm((PetscObject)dm),PETSC_ERR_SUP,"Complex values not supported");
#endif
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
tag = ((PetscObject)viewer)->tag;
ierr = PetscFOpen(comm,vtk->filename,"wb",&fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"<?xml version=\"1.0\"?>\n");CHKERRQ(ierr);
#if defined(PETSC_WORDS_BIGENDIAN)
ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"UnstructuredGrid\" version=\"0.1\" byte_order=\"BigEndian\">\n");CHKERRQ(ierr);
#else
ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"UnstructuredGrid\" version=\"0.1\" byte_order=\"LittleEndian\">\n");CHKERRQ(ierr);
#endif
ierr = PetscFPrintf(comm,fp," <UnstructuredGrid>\n");CHKERRQ(ierr);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetVTKCellHeight(dm, &cellHeight);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
if (cMax >= 0) cEnd = PetscMin(cEnd, cMax);
ierr = DMPlexGetStratumSize(dm, "vtk", 1, &numLabelCells);CHKERRQ(ierr);
hasLabel = numLabelCells > 0 ? PETSC_TRUE : PETSC_FALSE;
piece.nvertices = vEnd - vStart;
piece.ncells = 0;
piece.nconn = 0;
for (c = cStart; c < cEnd; ++c) {
PetscInt *closure = NULL;
PetscInt closureSize;
if (hasLabel) {
PetscInt value;
ierr = DMPlexGetLabelValue(dm, "vtk", c, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
ierr = DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
for (v = 0; v < closureSize*2; v += 2) {
if ((closure[v] >= vStart) && (closure[v] < vEnd)) piece.nconn++;
}
ierr = DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
piece.ncells++;
}
if (!rank) {ierr = PetscMalloc(size*sizeof(piece),&gpiece);CHKERRQ(ierr);}
ierr = MPI_Gather((PetscInt*)&piece,sizeof(piece)/sizeof(PetscInt),MPIU_INT,(PetscInt*)gpiece,sizeof(piece)/sizeof(PetscInt),MPIU_INT,0,comm);CHKERRQ(ierr);
/*
* Write file header
*/
if (!rank) {
PetscInt boffset = 0;
for (r=0; r<size; r++) {
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <Piece NumberOfPoints=\"%D\" NumberOfCells=\"%D\">\n",gpiece[r].nvertices,gpiece[r].ncells);CHKERRQ(ierr);
/* Coordinate positions */
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <Points>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"%s\" Name=\"Position\" NumberOfComponents=\"3\" format=\"appended\" offset=\"%D\" />\n",precision,boffset);CHKERRQ(ierr);
boffset += gpiece[r].nvertices*3*sizeof(PetscScalar) + sizeof(int);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </Points>\n");CHKERRQ(ierr);
/* Cell connectivity */
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <Cells>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"Int32\" Name=\"connectivity\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",boffset);CHKERRQ(ierr);
boffset += gpiece[r].nconn*sizeof(PetscInt) + sizeof(int);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"Int32\" Name=\"offsets\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",boffset);CHKERRQ(ierr);
boffset += gpiece[r].ncells*sizeof(PetscInt) + sizeof(int);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"UInt8\" Name=\"types\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",boffset);CHKERRQ(ierr);
boffset += gpiece[r].ncells*sizeof(unsigned char) + sizeof(int);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </Cells>\n");CHKERRQ(ierr);
/*
* Cell Data headers
*/
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <CellData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <DataArray type=\"Int32\" Name=\"Rank\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",boffset);CHKERRQ(ierr);
boffset += gpiece[r].ncells*sizeof(int) + sizeof(int);
/* all the vectors */
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
PetscInt bs,nfields,field;
const char *vecname = "";
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (((PetscObject)X)->name || link != vtk->link) { /* If the object is already named, use it. If it is past the first link, name it to disambiguate. */
ierr = PetscObjectGetName((PetscObject)X,&vecname);CHKERRQ(ierr);
}
ierr = PetscSectionGetDof(dm->defaultSection,cStart,&bs);CHKERRQ(ierr);
ierr = PetscSectionGetNumFields(dm->defaultSection,&nfields);CHKERRQ(ierr);
for (field=0,i=0; field<(nfields?nfields:1); field++) {
PetscInt fbs,j;
const char *fieldname = NULL;
char buf[256];
if (nfields) { /* We have user-defined fields/components */
ierr = PetscSectionGetFieldDof(dm->defaultSection,cStart,field,&fbs);CHKERRQ(ierr);
ierr = PetscSectionGetFieldName(dm->defaultSection,field,&fieldname);CHKERRQ(ierr);
} else fbs = bs; /* Say we have one field with 'bs' components */
if (!fieldname) {
ierr = PetscSNPrintf(buf,sizeof(buf),"CellField%D",field);CHKERRQ(ierr);
fieldname = buf;
}
for (j=0; j<fbs; j++) {
ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"%s%s.%D\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",precision,vecname,fieldname,j,boffset);CHKERRQ(ierr);
boffset += gpiece[r].ncells*sizeof(PetscScalar) + sizeof(int);
i++;
}
}
if (i != bs) SETERRQ2(comm,PETSC_ERR_PLIB,"Total number of field components %D != block size %D",i,bs);
}
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </CellData>\n");CHKERRQ(ierr);
/*
* Point Data headers
*/
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," <PointData>\n");CHKERRQ(ierr);
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
PetscInt bs,nfields,field;
const char *vecname = "";
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (((PetscObject)X)->name || link != vtk->link) { /* If the object is already named, use it. If it is past the first link, name it to disambiguate. */
ierr = PetscObjectGetName((PetscObject)X,&vecname);CHKERRQ(ierr);
}
ierr = PetscSectionGetDof(dm->defaultSection,vStart,&bs);CHKERRQ(ierr);
ierr = PetscSectionGetNumFields(dm->defaultSection,&nfields);CHKERRQ(ierr);
for (field=0,i=0; field<(nfields?nfields:1); field++) {
PetscInt fbs,j;
const char *fieldname = NULL;
char buf[256];
if (nfields) { /* We have user-defined fields/components */
ierr = PetscSectionGetFieldDof(dm->defaultSection,vStart,field,&fbs);CHKERRQ(ierr);
ierr = PetscSectionGetFieldName(dm->defaultSection,field,&fieldname);CHKERRQ(ierr);
} else fbs = bs; /* Say we have one field with 'bs' components */
if (!fieldname) {
ierr = PetscSNPrintf(buf,sizeof(buf),"PointField%D",field);CHKERRQ(ierr);
fieldname = buf;
}
for (j=0; j<fbs; j++) {
ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"%s%s.%D\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",precision,vecname,fieldname,j,boffset);CHKERRQ(ierr);
boffset += gpiece[r].nvertices*sizeof(PetscScalar) + sizeof(int);
}
}
}
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </PointData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(PETSC_COMM_SELF,fp," </Piece>\n");CHKERRQ(ierr);
}
}
ierr = PetscFPrintf(comm,fp," </UnstructuredGrid>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp," <AppendedData encoding=\"raw\">\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"_");CHKERRQ(ierr);
if (!rank) {
PetscInt maxsize = 0;
for (r=0; r<size; r++) {
maxsize = PetscMax(maxsize, (PetscInt) (gpiece[r].nvertices*3*sizeof(PetscScalar)));
maxsize = PetscMax(maxsize, (PetscInt) (gpiece[r].ncells*sizeof(PetscScalar)));
maxsize = PetscMax(maxsize, (PetscInt) (gpiece[r].nconn*sizeof(PetscVTKInt)));
}
ierr = PetscMalloc(maxsize,&buffer);CHKERRQ(ierr);
}
for (r=0; r<size; r++) {
if (r == rank) {
PetscInt nsend;
{ /* Position */
const PetscScalar *x;
PetscScalar *y = NULL;
Vec coords;
nsend = piece.nvertices*3;
ierr = DMGetCoordinatesLocal(dm,&coords);CHKERRQ(ierr);
ierr = VecGetArrayRead(coords,&x);CHKERRQ(ierr);
if (dim != 3) {
ierr = PetscMalloc(piece.nvertices*3*sizeof(PetscScalar),&y);CHKERRQ(ierr);
for (i=0; i<piece.nvertices; i++) {
y[i*3+0] = x[i*dim+0];
y[i*3+1] = (dim > 1) ? x[i*dim+1] : 0;
y[i*3+2] = 0;
}
}
ierr = TransferWrite(viewer,fp,r,0,y ? y : x,buffer,nsend,PETSC_SCALAR,tag);CHKERRQ(ierr);
ierr = PetscFree(y);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(coords,&x);CHKERRQ(ierr);
}
{ /* Connectivity, offsets, types */
PetscVTKInt *connectivity = NULL,*offsets;
PetscVTKType *types;
ierr = DMPlexGetVTKConnectivity(dm,&piece,&connectivity,&offsets,&types);CHKERRQ(ierr);
ierr = TransferWrite(viewer,fp,r,0,connectivity,buffer,piece.nconn,PETSC_INT32,tag);CHKERRQ(ierr);
ierr = TransferWrite(viewer,fp,r,0,offsets,buffer,piece.ncells,PETSC_INT32,tag);CHKERRQ(ierr);
ierr = TransferWrite(viewer,fp,r,0,types,buffer,piece.ncells,PETSC_UINT8,tag);CHKERRQ(ierr);
ierr = PetscFree3(connectivity,offsets,types);CHKERRQ(ierr);
}
{ /* Owners (cell data) */
PetscVTKInt *owners;
ierr = PetscMalloc(piece.ncells*sizeof(PetscVTKInt),&owners);CHKERRQ(ierr);
for (i=0; i<piece.ncells; i++) owners[i] = rank;
ierr = TransferWrite(viewer,fp,r,0,owners,buffer,piece.ncells,PETSC_INT32,tag);CHKERRQ(ierr);
ierr = PetscFree(owners);CHKERRQ(ierr);
}
/* Cell data */
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
const PetscScalar *x;
PetscScalar *y;
PetscInt bs;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
ierr = PetscSectionGetDof(dm->defaultSection,cStart,&bs);CHKERRQ(ierr);
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
ierr = PetscMalloc(piece.ncells*sizeof(PetscScalar),&y);CHKERRQ(ierr);
for (i=0; i<bs; i++) {
PetscInt cnt;
for (c=cStart,cnt=0; c<cEnd; c++) {
const PetscScalar *xpoint;
if (hasLabel) { /* Ignore some cells */
PetscInt value;
ierr = DMPlexGetLabelValue(dm, "vtk", c, &value);CHKERRQ(ierr);
if (value != 1) continue;
}
ierr = DMPlexPointLocalRead(dm,c,x,&xpoint);CHKERRQ(ierr);
y[cnt++] = xpoint[i];
}
if (cnt != piece.ncells) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Count does not match");
ierr = TransferWrite(viewer,fp,r,0,y,buffer,piece.ncells,PETSC_SCALAR,tag);CHKERRQ(ierr);
}
ierr = PetscFree(y);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
}
for (link=vtk->link; link; link=link->next) {
Vec X = (Vec)link->vec;
const PetscScalar *x;
PetscScalar *y;
PetscInt bs;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
ierr = PetscSectionGetDof(dm->defaultSection,vStart,&bs);CHKERRQ(ierr);
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
ierr = PetscMalloc(piece.nvertices*sizeof(PetscScalar),&y);CHKERRQ(ierr);
for (i=0; i<bs; i++) {
PetscInt cnt;
for (v=vStart,cnt=0; v<vEnd; v++) {
const PetscScalar *xpoint;
ierr = DMPlexPointLocalRead(dm,v,x,&xpoint);CHKERRQ(ierr);
y[cnt++] = xpoint[i];
}
if (cnt != piece.nvertices) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Count does not match");
ierr = TransferWrite(viewer,fp,r,0,y,buffer,piece.nvertices,PETSC_SCALAR,tag);CHKERRQ(ierr);
}
ierr = PetscFree(y);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
}
} else if (!rank) {
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].nvertices*3,PETSC_SCALAR,tag);CHKERRQ(ierr); /* positions */
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].nconn,PETSC_INT32,tag);CHKERRQ(ierr); /* connectivity */
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].ncells,PETSC_INT32,tag);CHKERRQ(ierr); /* offsets */
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].ncells,PETSC_UINT8,tag);CHKERRQ(ierr); /* types */
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].ncells,PETSC_INT32,tag);CHKERRQ(ierr); /* owner rank (cells) */
/* all cell data */
for (link=vtk->link; link; link=link->next) {
PetscInt bs;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
ierr = PetscSectionGetDof(dm->defaultSection,cStart,&bs);CHKERRQ(ierr);
for (i=0; i<bs; i++) {
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].ncells,PETSC_SCALAR,tag);CHKERRQ(ierr);
}
}
/* all point data */
for (link=vtk->link; link; link=link->next) {
PetscInt bs;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
ierr = PetscSectionGetDof(dm->defaultSection,vStart,&bs);CHKERRQ(ierr);
for (i=0; i<bs; i++) {
ierr = TransferWrite(viewer,fp,r,0,NULL,buffer,gpiece[r].nvertices,PETSC_SCALAR,tag);CHKERRQ(ierr);
}
}
}
}
ierr = PetscFree(gpiece);CHKERRQ(ierr);
ierr = PetscFree(buffer);CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"\n </AppendedData>\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fp,"</VTKFile>\n");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
