static PetscErrorCode ComputeMatrix(KSP ksp,Mat J,Mat jac,void *ctx)
{
AppCtx *user = (AppCtx*)ctx;
PetscErrorCode ierr;
PetscInt i,mx,xm,xs;
PetscScalar v[3],h,xlow,xhigh;
MatStencil row,col[3];
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xs,0,0,&xm,0,0);CHKERRQ(ierr);
h = 1.0/(mx-1);
for (i=xs; i<xs+xm; i++) {
row.i = i;
if (i==0 || i==mx-1) {
v[0] = 2.0;
ierr = MatSetValuesStencil(jac,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
} else {
xlow = h*(PetscReal)i - .5*h;
xhigh = xlow + h;
v[0] = (-1.0 - user->e*PetscSinScalar(2.0*PETSC_PI*user->k*xlow))/h;col[0].i = i-1;
v[1] = (2.0 + user->e*PetscSinScalar(2.0*PETSC_PI*user->k*xlow) + user->e*PetscSinScalar(2.0*PETSC_PI*user->k*xhigh))/h;col[1].i = row.i;
v[2] = (-1.0 - user->e*PetscSinScalar(2.0*PETSC_PI*user->k*xhigh))/h;col[2].i = i+1;
ierr = MatSetValuesStencil(jac,1,&row,3,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ComputeMatrix(KSP ksp,Mat J,Mat jac,void *ctx)
{
PetscErrorCode ierr;
PetscInt i,mx,xm,xs;
PetscScalar v[3],h;
MatStencil row,col[3];
DM da,shell;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&shell);CHKERRQ(ierr);
ierr = DMShellGetContext(shell,(void**)&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xs,0,0,&xm,0,0);CHKERRQ(ierr);
h = 1.0/(mx-1);
for (i=xs; i<xs+xm; i++) {
row.i = i;
if (i==0 || i==mx-1) {
v[0] = 2.0/h;
ierr = MatSetValuesStencil(jac,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
} else {
v[0] = (-1.0)/h;col[0].i = i-1;
v[1] = (2.0)/h;col[1].i = row.i;
v[2] = (-1.0)/h;col[2].i = i+1;
ierr = MatSetValuesStencil(jac,1,&row,3,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeJacobian(KSP ksp,Mat J, Mat jac,void *ctx)
{
PetscErrorCode ierr;
PetscInt i, j, M, N, xm, ym, xs, ys, num, numi, numj;
PetscScalar v[5], Hx, Hy, HydHx, HxdHy;
MatStencil row, col[5];
DM da;
MatNullSpace nullspace;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&M,&N,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(M);
Hy = 1.0 / (PetscReal)(N);
HxdHy = Hx/Hy;
HydHx = Hy/Hx;
ierr = DMDAGetCorners(da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
row.i = i; row.j = j;
if (i==0 || j==0 || i==M-1 || j==N-1) {
num=0; numi=0; numj=0;
if (j!=0) {
v[num] = -HxdHy; col[num].i = i; col[num].j = j-1;
num++; numj++;
}
if (i!=0) {
v[num] = -HydHx; col[num].i = i-1; col[num].j = j;
num++; numi++;
}
if (i!=M-1) {
v[num] = -HydHx; col[num].i = i+1; col[num].j = j;
num++; numi++;
}
if (j!=N-1) {
v[num] = -HxdHy; col[num].i = i; col[num].j = j+1;
num++; numj++;
}
v[num] = ((PetscReal)(numj)*HxdHy + (PetscReal)(numi)*HydHx); col[num].i = i; col[num].j = j;
num++;
ierr = MatSetValuesStencil(jac,1,&row,num,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else {
v[0] = -HxdHy; col[0].i = i; col[0].j = j-1;
v[1] = -HydHx; col[1].i = i-1; col[1].j = j;
v[2] = 2.0*(HxdHy + HydHx); col[2].i = i; col[2].j = j;
v[3] = -HydHx; col[3].i = i+1; col[3].j = j;
v[4] = -HxdHy; col[4].i = i; col[4].j = j+1;
ierr = MatSetValuesStencil(jac,1,&row,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);CHKERRQ(ierr);
ierr = MatSetNullSpace(J,nullspace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
We integrate over each cell
(i, j+1)----(i+1, j+1)
| \ |
| \ |
| \ |
| \ |
| \ |
| \ |
| \ |
(i, j)----(i+1, j)
*/
PetscErrorCode ComputeRHS(KSP ksp, Vec b, void *ctx)
{
UserContext *user = (UserContext*)ctx;
PetscScalar phi = user->phi;
PetscScalar *array;
PetscInt ne,nc,i;
const PetscInt *e;
PetscErrorCode ierr;
Vec blocal;
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
/* access a local vector with room for the ghost points */
ierr = DMGetLocalVector(da,&blocal);CHKERRQ(ierr);
ierr = VecGetArray(blocal, (PetscScalar**) &array);CHKERRQ(ierr);
/* access the list of elements on this processor and loop over them */
ierr = DMDAGetElements(da,&ne,&nc,&e);CHKERRQ(ierr);
for (i=0; i<ne; i++) {
/* this is nonsense, but set each nodal value to phi (will actually do integration over element */
array[e[3*i]] = phi;
array[e[3*i+1]] = phi;
array[e[3*i+2]] = phi;
}
ierr = VecRestoreArray(blocal, (PetscScalar**) &array);CHKERRQ(ierr);
ierr = DMDARestoreElements(da,&ne,&nc,&e);CHKERRQ(ierr);
/* add our partial sums over all processors into b */
ierr = DMLocalToGlobalBegin(da,blocal,ADD_VALUES,b);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,blocal, ADD_VALUES,b);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(da,&blocal);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeRHS(KSP ksp,Vec b,void *ctx)
{
PetscErrorCode ierr;
PetscInt i,j,k,mx,my,mz,xm,ym,zm,xs,ys,zs;
DM dm;
PetscScalar Hx,Hy,Hz,HxHydHz,HyHzdHx,HxHzdHy;
PetscScalar ***barray;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&dm);CHKERRQ(ierr);
ierr = DMDAGetInfo(dm,0,&mx,&my,&mz,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(mx-1); Hy = 1.0 / (PetscReal)(my-1); Hz = 1.0 / (PetscReal)(mz-1);
HxHydHz = Hx*Hy/Hz; HxHzdHy = Hx*Hz/Hy; HyHzdHx = Hy*Hz/Hx;
ierr = DMDAGetCorners(dm,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
ierr = DMDAVecGetArray(dm,b,&barray);CHKERRQ(ierr);
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
if (i==0 || j==0 || k==0 || i==mx-1 || j==my-1 || k==mz-1) {
barray[k][j][i] = 2.0*(HxHydHz + HxHzdHy + HyHzdHx);
} else {
barray[k][j][i] = Hx*Hy*Hz;
}
}
}
}
ierr = DMDAVecRestoreArray(dm,b,&barray);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeMatrix(KSP ksp,Mat J,Mat jac,MatStructure *str,void *ctx)
{
PetscErrorCode ierr;
PetscInt i,mx,xm,xs;
PetscScalar v[7],Hx;
MatStencil row,col[7];
PetscScalar lambda;
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = PetscMemzero(col,7*sizeof(MatStencil));CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 2.0*PETSC_PI / (PetscReal)(mx);
ierr = DMDAGetCorners(da,&xs,0,0,&xm,0,0);CHKERRQ(ierr);
lambda = 2.0*Hx;
for (i=xs; i<xs+xm; i++) {
row.i = i; row.j = 0; row.k = 0; row.c = 0;
v[0] = Hx; col[0].i = i; col[0].c = 0;
v[1] = lambda; col[1].i = i-1; col[1].c = 1;
v[2] = -lambda;col[2].i = i+1; col[2].c = 1;
ierr = MatSetValuesStencil(jac,1,&row,3,col,v,INSERT_VALUES);CHKERRQ(ierr);
row.i = i; row.j = 0; row.k = 0; row.c = 1;
v[0] = lambda; col[0].i = i-1; col[0].c = 0;
v[1] = Hx; col[1].i = i; col[1].c = 1;
v[2] = -lambda;col[2].i = i+1; col[2].c = 0;
ierr = MatSetValuesStencil(jac,1,&row,3,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(jac,PETSC_VIEWER_BINARY_(PETSC_COMM_SELF));CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ourkspcomputeoperators(KSP ksp,Mat A,Mat B,void *ctx)
{
PetscErrorCode ierr = 0;
DM dm;
DMKSP kdm;
ierr = KSPGetDM(ksp,&dm);CHKERRQ(ierr);
ierr = DMGetDMKSP(dm,&kdm);CHKERRQ(ierr);
(*(void (PETSC_STDCALL *)(KSP*,Mat*,Mat*,void*,PetscErrorCode*))(kdm->fortran_func_pointers[1]))(&ksp,&A,&B,ctx,&ierr);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode ourkspcomputeinitialguess(KSP ksp,Vec b,void *ctx)
{
PetscErrorCode ierr = 0;
DM dm;
DMKSP kdm;
ierr = KSPGetDM(ksp,&dm);CHKERRQ(ierr);
ierr = DMGetDMKSP(dm,&kdm);CHKERRQ(ierr);
(*(void (PETSC_STDCALL *)(KSP*,Vec*,void*,PetscErrorCode*))(kdm->fortran_func_pointers[2]))(&ksp,&b,ctx,&ierr);CHKERRQ(ierr);
return 0;
}
PetscErrorCode ComputeRHS(KSP ksp,Vec b,void *ctx)
{
PetscErrorCode ierr;
PetscInt mx;
PetscScalar h;
Vec x;
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMGetApplicationContext(da,&x);CHKERRQ(ierr);
h = 2.0*PETSC_PI/((mx));
ierr = VecCopy(x,b);CHKERRQ(ierr);
ierr = VecScale(b,h);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeRHS(KSP ksp,Vec b,void *ctx)
{
UserContext *user = (UserContext*)ctx;
PetscErrorCode ierr;
PetscInt i,j,mx,my,xm,ym,xs,ys;
PetscScalar Hx,Hy;
PetscScalar **array;
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);
CHKERRQ(ierr);
ierr = DMDAGetInfo(da, 0, &mx, &my, 0,0,0,0,0,0,0,0,0,0);
CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(mx-1);
Hy = 1.0 / (PetscReal)(my-1);
ierr = DMDAGetCorners(da,&xs,&ys,0,&xm,&ym,0);
CHKERRQ(ierr);
ierr = DMDAVecGetArray(da, b, &array);
CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
array[j][i] = PetscExpScalar(-((PetscReal)i*Hx)*((PetscReal)i*Hx)/user->nu)*PetscExpScalar(-((PetscReal)j*Hy)*((PetscReal)j*Hy)/user->nu)*Hx*Hy;
}
}
ierr = DMDAVecRestoreArray(da, b, &array);
CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);
CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);
CHKERRQ(ierr);
/* force right hand side to be consistent for singular matrix */
/* note this is really a hack, normally the model would provide you with a consistent right handside */
if (user->bcType == NEUMANN) {
MatNullSpace nullspace;
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);
CHKERRQ(ierr);
ierr = MatNullSpaceRemove(nullspace,b,PETSC_NULL);
CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);
CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*
We integrate over each cell
(i, j+1)----(i+1, j+1)
| \ |
| \ |
| \ |
| \ |
| \ |
| \ |
| \ |
(i, j)----(i+1, j)
However, the element stiffness matrix for the identity in linear elements is
1 /2 1 1\
- |1 2 1|
12 \1 1 2/
no matter what the shape of the triangle. The Laplacian stiffness matrix is
1 / (x_2 - x_1)^2 + (y_2 - y_1)^2 -(x_2 - x_0)(x_2 - x_1) - (y_2 - y_1)(y_2 - y_0) (x_1 - x_0)(x_2 - x_1) + (y_1 - y_0)(y_2 - y_1)\
- |-(x_2 - x_0)(x_2 - x_1) - (y_2 - y_1)(y_2 - y_0) (x_2 - x_0)^2 + (y_2 - y_0)^2 -(x_1 - x_0)(x_2 - x_0) - (y_1 - y_0)(y_2 - y_0)|
A \ (x_1 - x_0)(x_2 - x_1) + (y_1 - y_0)(y_2 - y_1) -(x_1 - x_0)(x_2 - x_0) - (y_1 - y_0)(y_2 - y_0) (x_1 - x_0)^2 + (y_1 - y_0)^2 /
where A is the area of the triangle, and (x_i, y_i) is its i'th vertex.
*/
PetscErrorCode ComputeMatrix(KSP ksp, Mat J, Mat jac, MatStructure *flag,void *ctx)
{
UserContext *user = (UserContext*)ctx;
/* not being used!
PetscScalar identity[9] = {0.16666666667, 0.08333333333, 0.08333333333,
0.08333333333, 0.16666666667, 0.08333333333,
0.08333333333, 0.08333333333, 0.16666666667};
*/
PetscScalar values[3][3];
PetscInt idx[3];
PetscScalar hx, hy, hx2, hy2, area,phi_dt2;
PetscInt i,mx,my,xm,ym,xs,ys;
PetscInt ne,nc;
const PetscInt *e;
PetscErrorCode ierr;
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da, 0, &mx, &my, 0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DMDAGetCorners(da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
hx = 1.0 / (mx-1);
hy = 1.0 / (my-1);
area = 0.5*hx*hy;
phi_dt2 = user->phi*user->dt*user->dt;
hx2 = hx*hx/area*phi_dt2;
hy2 = hy*hy/area*phi_dt2;
/* initially all elements have identical geometry so all element stiffness are identical */
values[0][0] = hx2 + hy2; values[0][1] = -hy2; values[0][2] = -hx2;
values[1][0] = -hy2; values[1][1] = hy2; values[1][2] = 0.0;
values[2][0] = -hx2; values[2][1] = 0.0; values[2][2] = hx2;
ierr = DMDAGetElements(da,&ne,&nc,&e);CHKERRQ(ierr);
for (i=0; i<ne; i++) {
idx[0] = e[3*i];
idx[1] = e[3*i+1];
idx[2] = e[3*i+2];
ierr = MatSetValuesLocal(jac,3,idx,3,idx,(PetscScalar*)values,ADD_VALUES);CHKERRQ(ierr);
}
ierr = DMDARestoreElements(da,&ne,&nc,&e);CHKERRQ(ierr);
ierr = MatAssemblyBegin(jac, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ComputeRHS(KSP ksp,Vec b,void *ctx)
{
PetscErrorCode ierr;
PetscInt mx,idx[2];
PetscScalar h,v[2];
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
h = 1.0/((mx-1));
ierr = VecSet(b,h);CHKERRQ(ierr);
idx[0] = 0; idx[1] = mx -1;
v[0] = v[1] = 0.0;
ierr = VecSetValues(b,2,idx,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeRHS(KSP ksp,Vec b,void *ctx)
{
UserContext *user = (UserContext*)ctx;
PetscErrorCode ierr;
PetscInt i, j, M, N, xm ,ym ,xs, ys;
PetscScalar Hx, Hy, pi, uu, tt;
PetscScalar **array;
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da, 0, &M, &N, 0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
uu = user->uu; tt = user->tt;
pi = 4*atan(1.0);
Hx = 1.0/(PetscReal)(M);
Hy = 1.0/(PetscReal)(N);
ierr = DMDAGetCorners(da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr); // Fine grid
//printf(" M N: %d %d; xm ym: %d %d; xs ys: %d %d\n",M,N,xm,ym,xs,ys);
ierr = DMDAVecGetArray(da, b, &array);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++){
for (i=xs; i<xs+xm; i++){
array[j][i] = -PetscCosScalar(uu*pi*((PetscReal)i+0.5)*Hx)*cos(tt*pi*((PetscReal)j+0.5)*Hy)*Hx*Hy;
}
}
ierr = DMDAVecRestoreArray(da, b, &array);CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);CHKERRQ(ierr);
/* force right hand side to be consistent for singular matrix */
/* note this is really a hack, normally the model would provide you with a consistent right handside */
if (user->bcType == NEUMANN) {
MatNullSpace nullspace;
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);CHKERRQ(ierr);
ierr = MatNullSpaceRemove(nullspace,b,PETSC_NULL);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode ComputeMatrix(KSP ksp,Mat jac,Mat B,MatStructure *stflg,void *ctx)
{
DM da;
PetscErrorCode ierr;
PetscInt i,j,k,mx,my,mz,xm,ym,zm,xs,ys,zs;
PetscScalar v[7],Hx,Hy,Hz,HxHydHz,HyHzdHx,HxHzdHy;
MatStencil row,col[7];
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,&my,&mz,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(mx-1); Hy = 1.0 / (PetscReal)(my-1); Hz = 1.0 / (PetscReal)(mz-1);
HxHydHz = Hx*Hy/Hz; HxHzdHy = Hx*Hz/Hy; HyHzdHx = Hy*Hz/Hx;
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
row.i = i; row.j = j; row.k = k;
if (i==0 || j==0 || k==0 || i==mx-1 || j==my-1 || k==mz-1) {
v[0] = 2.0*(HxHydHz + HxHzdHy + HyHzdHx);
ierr = MatSetValuesStencil(B,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
} else {
v[0] = -HxHydHz;col[0].i = i; col[0].j = j; col[0].k = k-1;
v[1] = -HxHzdHy;col[1].i = i; col[1].j = j-1; col[1].k = k;
v[2] = -HyHzdHx;col[2].i = i-1; col[2].j = j; col[2].k = k;
v[3] = 2.0*(HxHydHz + HxHzdHy + HyHzdHx);col[3].i = row.i; col[3].j = row.j; col[3].k = row.k;
v[4] = -HyHzdHx;col[4].i = i+1; col[4].j = j; col[4].k = k;
v[5] = -HxHzdHy;col[5].i = i; col[5].j = j+1; col[5].k = k;
v[6] = -HxHydHz;col[6].i = i; col[6].j = j; col[6].k = k+1;
ierr = MatSetValuesStencil(B,1,&row,7,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
}
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
*stflg = SAME_NONZERO_PATTERN;
PetscFunctionReturn(0);
}
PetscErrorCode ComputeRHS(KSP ksp,Vec b,void *ctx)
{
PetscErrorCode ierr;
PetscInt i,j,k,mx,my,mz,xm,ym,zm,xs,ys,zs;
PetscScalar Hx,Hy,Hz;
PetscScalar ***array;
DM da;
MatNullSpace nullspace;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da, 0, &mx, &my, &mz, 0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(mx);
Hy = 1.0 / (PetscReal)(my);
Hz = 1.0 / (PetscReal)(mz);
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
ierr = DMDAVecGetArray(da, b, &array);CHKERRQ(ierr);
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
array[k][j][i] = 12 * PETSC_PI * PETSC_PI
* PetscCosScalar(2*PETSC_PI*(((PetscReal)i+0.5)*Hx))
* PetscCosScalar(2*PETSC_PI*(((PetscReal)j+0.5)*Hy))
* PetscCosScalar(2*PETSC_PI*(((PetscReal)k+0.5)*Hz))
* Hx * Hy * Hz;
}
}
}
ierr = DMDAVecRestoreArray(da, b, &array);CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);CHKERRQ(ierr);
/* force right hand side to be consistent for singular matrix */
/* note this is really a hack, normally the model would provide you with a consistent right handside */
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);CHKERRQ(ierr);
ierr = MatNullSpaceRemove(nullspace,b);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeMatrix(KSP ksp,Mat J,Mat jac,void *ctx)
{
UserContext *user = (UserContext*)ctx;
PetscReal centerRho;
PetscErrorCode ierr;
PetscInt i,j,mx,my,xm,ym,xs,ys;
PetscScalar v[5];
PetscReal Hx,Hy,HydHx,HxdHy,rho;
MatStencil row, col[5];
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
centerRho = user->rho;
ierr = DMDAGetInfo(da,0,&mx,&my,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(mx-1);
Hy = 1.0 / (PetscReal)(my-1);
HxdHy = Hx/Hy;
HydHx = Hy/Hx;
ierr = DMDAGetCorners(da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
row.i = i; row.j = j;
ierr = ComputeRho(i, j, mx, my, centerRho, &rho);CHKERRQ(ierr);
if (i==0 || j==0 || i==mx-1 || j==my-1) {
if (user->bcType == DIRICHLET) {
v[0] = 2.0*rho*(HxdHy + HydHx);
ierr = MatSetValuesStencil(jac,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
} else if (user->bcType == NEUMANN) {
PetscInt numx = 0, numy = 0, num = 0;
if (j!=0) {
v[num] = -rho*HxdHy; col[num].i = i; col[num].j = j-1;
numy++; num++;
}
if (i!=0) {
v[num] = -rho*HydHx; col[num].i = i-1; col[num].j = j;
numx++; num++;
}
if (i!=mx-1) {
v[num] = -rho*HydHx; col[num].i = i+1; col[num].j = j;
numx++; num++;
}
if (j!=my-1) {
v[num] = -rho*HxdHy; col[num].i = i; col[num].j = j+1;
numy++; num++;
}
v[num] = numx*rho*HydHx + numy*rho*HxdHy; col[num].i = i; col[num].j = j;
num++;
ierr = MatSetValuesStencil(jac,1,&row,num,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
} else {
v[0] = -rho*HxdHy; col[0].i = i; col[0].j = j-1;
v[1] = -rho*HydHx; col[1].i = i-1; col[1].j = j;
v[2] = 2.0*rho*(HxdHy + HydHx); col[2].i = i; col[2].j = j;
v[3] = -rho*HydHx; col[3].i = i+1; col[3].j = j;
v[4] = -rho*HxdHy; col[4].i = i; col[4].j = j+1;
ierr = MatSetValuesStencil(jac,1,&row,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (user->bcType == NEUMANN) {
MatNullSpace nullspace;
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);CHKERRQ(ierr);
ierr = MatSetNullSpace(jac,nullspace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode computeRHS3D(KSP ksp, Vec b, void* ctx){
FlowField & flowField = ((PetscUserCtx*)ctx)->getFlowField();
Parameters & parameters = ((PetscUserCtx*)ctx)->getParameters();
ScalarField & RHS = flowField.getRHS();
PetscUserCtx * context = ((PetscUserCtx*)ctx);
IntScalarField & flags = flowField.getFlags();
int *limitsX, *limitsY, *limitsZ;
((PetscUserCtx*)ctx)->getLimits(&limitsX, &limitsY, &limitsZ);
PetscInt i, j, k;
PetscInt Nx = parameters.geometry.sizeX + 2,
Ny = parameters.geometry.sizeY + 2,
Nz = parameters.geometry.sizeZ + 2;
PetscScalar*** array;
DM da;
KSPGetDM(ksp, &da);
DMDAVecGetArray(da, b, &array);
// Notice that we're covering the whole surface, including corners and edges
// Also, the actual value is taking from the parameters.
// Left wall
if (context->setAsBoundary & LEFT_WALL_BIT){
if (parameters.simulation.scenario == "pressure-channel"){
for (k = limitsZ[0]; k < limitsZ[1]; k++){
for (j = limitsY[0]; j < limitsY[1]; j++){
array[k][j][0] = RHS.getScalar(0,j,k);
}
}
} else {
for (k = limitsZ[0]; k < limitsZ[1]; k++){
for (j = limitsY[0]; j < limitsY[1]; j++){
array[k][j][0] = 0;
}
}
}
}
// Right wall
if (context->setAsBoundary & RIGHT_WALL_BIT){
for (k = limitsZ[0]; k < limitsZ[1]; k++){
for (j = limitsY[0]; j < limitsY[1]; j++){
array[k][j][Nx-1] = 0;
}
}
}
// Bottom wall
if (context->setAsBoundary & BOTTOM_WALL_BIT){
for (k = limitsZ[0]; k < limitsZ[1]; k++){
for (i = limitsX[0]; i < limitsX[1]; i++){
array[k][0][i] = 0;
}
}
}
// Top wall
if (context->setAsBoundary & TOP_WALL_BIT){
for (k = limitsZ[0]; k < limitsZ[1]; k++){
for (i = limitsX[0]; i < limitsX[1]; i++){
array[k][Ny-1][i] = 0;
}
}
}
// Front wall
if (context->setAsBoundary & FRONT_WALL_BIT){
for (j = limitsY[0]; j < limitsY[1]; j++){
for (i = limitsX[0]; i < limitsX[1]; i++){
array[0][j][i] = 0;
}
}
}
// Back wall
if (context->setAsBoundary & BACK_WALL_BIT){
for (j = limitsY[0]; j < limitsY[1]; j++){
for (i = limitsX[0]; i < limitsX[1]; i++){
array[Nz-1][j][i] = 0;
}
}
}
// Fill the internal nodes. We already have the values
for (k = limitsZ[0]; k < limitsZ[1]; k++){
for (j = limitsY[0]; j < limitsY[1]; j++){
for (i = limitsX[0]; i < limitsX[1]; i++){
const int obstacle = flags.getValue(i-limitsX[0]+2, j-limitsY[0]+2, k-limitsZ[0]+2);
if ((obstacle & OBSTACLE_SELF) == 0) { // If this is a fluid cell
array[k][j][i] = RHS.getScalar(i-limitsX[0]+2, j-limitsY[0]+2, k-limitsZ[0]+2);
} else {
array[k][j][i] = 0.0;
}
}
}
}
DMDAVecRestoreArray(da, b, &array);
VecAssemblyBegin(b);
VecAssemblyEnd(b);
return 0;
}
PetscErrorCode computeRHS2D(KSP ksp, Vec b, void* ctx){
FlowField & flowField = ((PetscUserCtx*)ctx)->getFlowField();
Parameters & parameters = ((PetscUserCtx*)ctx)->getParameters();
PetscUserCtx * context = ((PetscUserCtx*)ctx);
int *limitsX, *limitsY, *limitsZ;
((PetscUserCtx*)ctx)->getLimits(&limitsX, &limitsY, &limitsZ);
IntScalarField & flags = context->getFlowField().getFlags();
ScalarField & RHS = flowField.getRHS();
PetscInt i, j;
PetscInt Nx = parameters.geometry.sizeX + 2,
Ny = parameters.geometry.sizeY + 2;
PetscScalar** array;
DM da;
KSPGetDM(ksp, &da);
DMDAVecGetArray(da, b, &array);
// Iteration domains are going to be set and the values on the global boundary set when
// necessary
// Check left wall
if (context->setAsBoundary & LEFT_WALL_BIT){
if (parameters.simulation.scenario == "pressure-channel"){
for (j = limitsY[0]; j < limitsY[1]; j++){
array[j][0] = RHS.getScalar(0,j);
}
} else {
for (j = limitsY[0]; j < limitsY[1]; j++){
array[j][0] = 0;
}
}
}
// Check right wall
if (context->setAsBoundary & RIGHT_WALL_BIT){
for (j = limitsY[0]; j < limitsY[1]; j++){
array[j][Nx-1] = 0;
}
}
// Check bottom wall
if (context->setAsBoundary & BOTTOM_WALL_BIT){
for (i = limitsX[0]; i < limitsX[1]; i++){
array[0][i] = 0;
}
}
// Check top wall
if (context->setAsBoundary & TOP_WALL_BIT){
for (i = limitsX[0]; i < limitsX[1]; i++){
array[Ny-1][i] = 0;
}
}
// Fill the internal nodes. We already have the values
for (j = limitsY[0]; j < limitsY[1]; j++){
for (i = limitsX[0]; i < limitsX[1]; i++){
const int obstacle = flags.getValue(i-limitsX[0]+2, j-limitsY[0]+2);
if ((obstacle & OBSTACLE_SELF) == 0) { // If this is a fluid cell
array[j][i] = RHS.getScalar(i-limitsX[0]+2, j-limitsY[0]+2);
} else {
array[j][i] = 0.0;
}
}
}
DMDAVecRestoreArray(da, b, &array);
VecAssemblyBegin(b);
VecAssemblyEnd(b);
return 0;
}
PetscErrorCode ComputeMatrix(KSP ksp, Mat J,Mat jac, void *ctx)
{
PetscErrorCode ierr;
PetscInt i,j,k,mx,my,mz,xm,ym,zm,xs,ys,zs,num, numi, numj, numk;
PetscScalar v[7],Hx,Hy,Hz,HyHzdHx,HxHzdHy,HxHydHz;
MatStencil row, col[7];
DM da;
MatNullSpace nullspace;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,&my,&mz,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(mx);
Hy = 1.0 / (PetscReal)(my);
Hz = 1.0 / (PetscReal)(mz);
HyHzdHx = Hy*Hz/Hx;
HxHzdHy = Hx*Hz/Hy;
HxHydHz = Hx*Hy/Hz;
ierr = DMDAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
for (k=zs; k<zs+zm; k++) {
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
row.i = i; row.j = j; row.k = k;
if (i==0 || j==0 || k==0 || i==mx-1 || j==my-1 || k==mz-1) {
num = 0; numi=0; numj=0; numk=0;
if (k!=0) {
v[num] = -HxHydHz;
col[num].i = i;
col[num].j = j;
col[num].k = k-1;
num++; numk++;
}
if (j!=0) {
v[num] = -HxHzdHy;
col[num].i = i;
col[num].j = j-1;
col[num].k = k;
num++; numj++;
}
if (i!=0) {
v[num] = -HyHzdHx;
col[num].i = i-1;
col[num].j = j;
col[num].k = k;
num++; numi++;
}
if (i!=mx-1) {
v[num] = -HyHzdHx;
col[num].i = i+1;
col[num].j = j;
col[num].k = k;
num++; numi++;
}
if (j!=my-1) {
v[num] = -HxHzdHy;
col[num].i = i;
col[num].j = j+1;
col[num].k = k;
num++; numj++;
}
if (k!=mz-1) {
v[num] = -HxHydHz;
col[num].i = i;
col[num].j = j;
col[num].k = k+1;
num++; numk++;
}
v[num] = (PetscReal)(numk)*HxHydHz + (PetscReal)(numj)*HxHzdHy + (PetscReal)(numi)*HyHzdHx;
col[num].i = i; col[num].j = j; col[num].k = k;
num++;
ierr = MatSetValuesStencil(jac,1,&row,num,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else {
v[0] = -HxHydHz; col[0].i = i; col[0].j = j; col[0].k = k-1;
v[1] = -HxHzdHy; col[1].i = i; col[1].j = j-1; col[1].k = k;
v[2] = -HyHzdHx; col[2].i = i-1; col[2].j = j; col[2].k = k;
v[3] = 2.0*(HyHzdHx + HxHzdHy + HxHydHz); col[3].i = i; col[3].j = j; col[3].k = k;
v[4] = -HyHzdHx; col[4].i = i+1; col[4].j = j; col[4].k = k;
v[5] = -HxHzdHy; col[5].i = i; col[5].j = j+1; col[5].k = k;
v[6] = -HxHydHz; col[6].i = i; col[6].j = j; col[6].k = k+1;
ierr = MatSetValuesStencil(jac,1,&row,7,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);CHKERRQ(ierr);
ierr = MatSetNullSpace(jac,nullspace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeMatrix(KSP ksp, Mat J,Mat jac,MatStructure *str, void *ctx)
{
UserContext *user = (UserContext*)ctx;
PetscErrorCode ierr;
PetscInt i,j,mx,my,xm,ym,xs,ys,num, numi, numj;
PetscScalar v[5],Hx,Hy,HydHx,HxdHy;
MatStencil row, col[5];
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&mx,&my,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(mx);
Hy = 1.0 / (PetscReal)(my);
HxdHy = Hx/Hy;
HydHx = Hy/Hx;
ierr = DMDAGetCorners(da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
row.i = i; row.j = j;
if (i==0 || j==0 || i==mx-1 || j==my-1) {
if (user->bcType == DIRICHLET) {
v[0] = 2.0*(HxdHy + HydHx);
ierr = MatSetValuesStencil(jac,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Dirichlet boundary conditions not supported !\n");
} else if (user->bcType == NEUMANN) {
num = 0; numi=0; numj=0;
if (j!=0) {
v[num] = -HxdHy;
col[num].i = i;
col[num].j = j-1;
num++; numj++;
}
if (i!=0) {
v[num] = -HydHx;
col[num].i = i-1;
col[num].j = j;
num++; numi++;
}
if (i!=mx-1) {
v[num] = -HydHx;
col[num].i = i+1;
col[num].j = j;
num++; numi++;
}
if (j!=my-1) {
v[num] = -HxdHy;
col[num].i = i;
col[num].j = j+1;
num++; numj++;
}
v[num] = (PetscReal)(numj)*HxdHy + (PetscReal)(numi)*HydHx; col[num].i = i; col[num].j = j;
num++;
ierr = MatSetValuesStencil(jac,1,&row,num,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
} else {
v[0] = -HxdHy; col[0].i = i; col[0].j = j-1;
v[1] = -HydHx; col[1].i = i-1; col[1].j = j;
v[2] = 2.0*(HxdHy + HydHx); col[2].i = i; col[2].j = j;
v[3] = -HydHx; col[3].i = i+1; col[3].j = j;
v[4] = -HxdHy; col[4].i = i; col[4].j = j+1;
ierr = MatSetValuesStencil(jac,1,&row,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (user->bcType == NEUMANN) {
MatNullSpace nullspace;
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);CHKERRQ(ierr);
ierr = MatSetNullSpace(jac,nullspace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode PCSetUp_MG(PC pc)
{
PC_MG *mg = (PC_MG*)pc->data;
PC_MG_Levels **mglevels = mg->levels;
PetscErrorCode ierr;
PetscInt i,n = mglevels[0]->levels;
PC cpc;
PetscBool dump = PETSC_FALSE,opsset,use_amat,missinginterpolate = PETSC_FALSE;
Mat dA,dB;
Vec tvec;
DM *dms;
PetscViewer viewer = 0;
PetscFunctionBegin;
/* FIX: Move this to PCSetFromOptions_MG? */
if (mg->usedmfornumberoflevels) {
PetscInt levels;
ierr = DMGetRefineLevel(pc->dm,&levels);CHKERRQ(ierr);
levels++;
if (levels > n) { /* the problem is now being solved on a finer grid */
ierr = PCMGSetLevels(pc,levels,NULL);CHKERRQ(ierr);
n = levels;
ierr = PCSetFromOptions(pc);CHKERRQ(ierr); /* it is bad to call this here, but otherwise will never be called for the new hierarchy */
mglevels = mg->levels;
}
}
ierr = KSPGetPC(mglevels[0]->smoothd,&cpc);CHKERRQ(ierr);
/* If user did not provide fine grid operators OR operator was not updated since last global KSPSetOperators() */
/* so use those from global PC */
/* Is this what we always want? What if user wants to keep old one? */
ierr = KSPGetOperatorsSet(mglevels[n-1]->smoothd,NULL,&opsset);CHKERRQ(ierr);
if (opsset) {
Mat mmat;
ierr = KSPGetOperators(mglevels[n-1]->smoothd,NULL,&mmat);CHKERRQ(ierr);
if (mmat == pc->pmat) opsset = PETSC_FALSE;
}
if (!opsset) {
ierr = PCGetUseAmat(pc,&use_amat);CHKERRQ(ierr);
if(use_amat){
ierr = PetscInfo(pc,"Using outer operators to define finest grid operator \n because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[n-1]->smoothd,pc->mat,pc->pmat);CHKERRQ(ierr);
}
else {
ierr = PetscInfo(pc,"Using matrix (pmat) operators to define finest grid operator \n because PCMGGetSmoother(pc,nlevels-1,&ksp);KSPSetOperators(ksp,...); was not called.\n");CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[n-1]->smoothd,pc->pmat,pc->pmat);CHKERRQ(ierr);
}
}
for (i=n-1; i>0; i--) {
if (!(mglevels[i]->interpolate || mglevels[i]->restrct)) {
missinginterpolate = PETSC_TRUE;
continue;
}
}
/*
Skipping if user has provided all interpolation/restriction needed (since DM might not be able to produce them (when coming from SNES/TS)
Skipping for galerkin==2 (externally managed hierarchy such as ML and GAMG). Cleaner logic here would be great. Wrap ML/GAMG as DMs?
*/
if (missinginterpolate && pc->dm && mg->galerkin != 2 && !pc->setupcalled) {
/* construct the interpolation from the DMs */
Mat p;
Vec rscale;
ierr = PetscMalloc1(n,&dms);CHKERRQ(ierr);
dms[n-1] = pc->dm;
/* Separately create them so we do not get DMKSP interference between levels */
for (i=n-2; i>-1; i--) {ierr = DMCoarsen(dms[i+1],MPI_COMM_NULL,&dms[i]);CHKERRQ(ierr);}
for (i=n-2; i>-1; i--) {
DMKSP kdm;
PetscBool dmhasrestrict;
ierr = KSPSetDM(mglevels[i]->smoothd,dms[i]);CHKERRQ(ierr);
if (mg->galerkin) {ierr = KSPSetDMActive(mglevels[i]->smoothd,PETSC_FALSE);CHKERRQ(ierr);}
ierr = DMGetDMKSPWrite(dms[i],&kdm);CHKERRQ(ierr);
/* Ugly hack so that the next KSPSetUp() will use the RHS that we set. A better fix is to change dmActive to take
* a bitwise OR of computing the matrix, RHS, and initial iterate. */
kdm->ops->computerhs = NULL;
kdm->rhsctx = NULL;
if (!mglevels[i+1]->interpolate) {
ierr = DMCreateInterpolation(dms[i],dms[i+1],&p,&rscale);CHKERRQ(ierr);
ierr = PCMGSetInterpolation(pc,i+1,p);CHKERRQ(ierr);
if (rscale) {ierr = PCMGSetRScale(pc,i+1,rscale);CHKERRQ(ierr);}
ierr = VecDestroy(&rscale);CHKERRQ(ierr);
ierr = MatDestroy(&p);CHKERRQ(ierr);
}
ierr = DMHasCreateRestriction(dms[i],&dmhasrestrict);CHKERRQ(ierr);
if (dmhasrestrict && !mglevels[i+1]->restrct){
ierr = DMCreateRestriction(dms[i],dms[i+1],&p);CHKERRQ(ierr);
ierr = PCMGSetRestriction(pc,i+1,p);CHKERRQ(ierr);
ierr = MatDestroy(&p);CHKERRQ(ierr);
}
}
for (i=n-2; i>-1; i--) {ierr = DMDestroy(&dms[i]);CHKERRQ(ierr);}
ierr = PetscFree(dms);CHKERRQ(ierr);
}
if (pc->dm && !pc->setupcalled) {
/* finest smoother also gets DM but it is not active, independent of whether galerkin==2 */
ierr = KSPSetDM(mglevels[n-1]->smoothd,pc->dm);CHKERRQ(ierr);
ierr = KSPSetDMActive(mglevels[n-1]->smoothd,PETSC_FALSE);CHKERRQ(ierr);
}
if (mg->galerkin == 1) {
Mat B;
/* currently only handle case where mat and pmat are the same on coarser levels */
ierr = KSPGetOperators(mglevels[n-1]->smoothd,&dA,&dB);CHKERRQ(ierr);
if (!pc->setupcalled) {
for (i=n-2; i>-1; i--) {
if (!mglevels[i+1]->restrct && !mglevels[i+1]->interpolate) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must provide interpolation or restriction for each MG level except level 0");
if (!mglevels[i+1]->interpolate) {
ierr = PCMGSetInterpolation(pc,i+1,mglevels[i+1]->restrct);CHKERRQ(ierr);
}
if (!mglevels[i+1]->restrct) {
ierr = PCMGSetRestriction(pc,i+1,mglevels[i+1]->interpolate);CHKERRQ(ierr);
}
if (mglevels[i+1]->interpolate == mglevels[i+1]->restrct) {
ierr = MatPtAP(dB,mglevels[i+1]->interpolate,MAT_INITIAL_MATRIX,1.0,&B);CHKERRQ(ierr);
} else {
ierr = MatMatMatMult(mglevels[i+1]->restrct,dB,mglevels[i+1]->interpolate,MAT_INITIAL_MATRIX,1.0,&B);CHKERRQ(ierr);
}
ierr = KSPSetOperators(mglevels[i]->smoothd,B,B);CHKERRQ(ierr);
if (i != n-2) {ierr = PetscObjectDereference((PetscObject)dB);CHKERRQ(ierr);}
dB = B;
}
if (n > 1) {ierr = PetscObjectDereference((PetscObject)dB);CHKERRQ(ierr);}
} else {
for (i=n-2; i>-1; i--) {
if (!mglevels[i+1]->restrct && !mglevels[i+1]->interpolate) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must provide interpolation or restriction for each MG level except level 0");
if (!mglevels[i+1]->interpolate) {
ierr = PCMGSetInterpolation(pc,i+1,mglevels[i+1]->restrct);CHKERRQ(ierr);
}
if (!mglevels[i+1]->restrct) {
ierr = PCMGSetRestriction(pc,i+1,mglevels[i+1]->interpolate);CHKERRQ(ierr);
}
ierr = KSPGetOperators(mglevels[i]->smoothd,NULL,&B);CHKERRQ(ierr);
if (mglevels[i+1]->interpolate == mglevels[i+1]->restrct) {
ierr = MatPtAP(dB,mglevels[i+1]->interpolate,MAT_REUSE_MATRIX,1.0,&B);CHKERRQ(ierr);
} else {
ierr = MatMatMatMult(mglevels[i+1]->restrct,dB,mglevels[i+1]->interpolate,MAT_REUSE_MATRIX,1.0,&B);CHKERRQ(ierr);
}
ierr = KSPSetOperators(mglevels[i]->smoothd,B,B);CHKERRQ(ierr);
dB = B;
}
}
} else if (!mg->galerkin && pc->dm && pc->dm->x) {
/* need to restrict Jacobian location to coarser meshes for evaluation */
for (i=n-2; i>-1; i--) {
Mat R;
Vec rscale;
if (!mglevels[i]->smoothd->dm->x) {
Vec *vecs;
ierr = KSPCreateVecs(mglevels[i]->smoothd,1,&vecs,0,NULL);CHKERRQ(ierr);
mglevels[i]->smoothd->dm->x = vecs[0];
ierr = PetscFree(vecs);CHKERRQ(ierr);
}
ierr = PCMGGetRestriction(pc,i+1,&R);CHKERRQ(ierr);
ierr = PCMGGetRScale(pc,i+1,&rscale);CHKERRQ(ierr);
ierr = MatRestrict(R,mglevels[i+1]->smoothd->dm->x,mglevels[i]->smoothd->dm->x);CHKERRQ(ierr);
ierr = VecPointwiseMult(mglevels[i]->smoothd->dm->x,mglevels[i]->smoothd->dm->x,rscale);CHKERRQ(ierr);
}
}
if (!mg->galerkin && pc->dm) {
for (i=n-2; i>=0; i--) {
DM dmfine,dmcoarse;
Mat Restrict,Inject;
Vec rscale;
ierr = KSPGetDM(mglevels[i+1]->smoothd,&dmfine);CHKERRQ(ierr);
ierr = KSPGetDM(mglevels[i]->smoothd,&dmcoarse);CHKERRQ(ierr);
ierr = PCMGGetRestriction(pc,i+1,&Restrict);CHKERRQ(ierr);
ierr = PCMGGetRScale(pc,i+1,&rscale);CHKERRQ(ierr);
Inject = NULL; /* Callback should create it if it needs Injection */
ierr = DMRestrict(dmfine,Restrict,rscale,Inject,dmcoarse);CHKERRQ(ierr);
}
}
if (!pc->setupcalled) {
for (i=0; i<n; i++) {
ierr = KSPSetFromOptions(mglevels[i]->smoothd);CHKERRQ(ierr);
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu && (mglevels[i]->smoothu != mglevels[i]->smoothd)) {
ierr = KSPSetFromOptions(mglevels[i]->smoothu);CHKERRQ(ierr);
}
}
/* insure that if either interpolation or restriction is set the other other one is set */
for (i=1; i<n; i++) {
ierr = PCMGGetInterpolation(pc,i,NULL);CHKERRQ(ierr);
ierr = PCMGGetRestriction(pc,i,NULL);CHKERRQ(ierr);
}
for (i=0; i<n-1; i++) {
if (!mglevels[i]->b) {
Vec *vec;
ierr = KSPCreateVecs(mglevels[i]->smoothd,1,&vec,0,NULL);CHKERRQ(ierr);
ierr = PCMGSetRhs(pc,i,*vec);CHKERRQ(ierr);
ierr = VecDestroy(vec);CHKERRQ(ierr);
ierr = PetscFree(vec);CHKERRQ(ierr);
}
if (!mglevels[i]->r && i) {
ierr = VecDuplicate(mglevels[i]->b,&tvec);CHKERRQ(ierr);
ierr = PCMGSetR(pc,i,tvec);CHKERRQ(ierr);
ierr = VecDestroy(&tvec);CHKERRQ(ierr);
}
if (!mglevels[i]->x) {
ierr = VecDuplicate(mglevels[i]->b,&tvec);CHKERRQ(ierr);
ierr = PCMGSetX(pc,i,tvec);CHKERRQ(ierr);
ierr = VecDestroy(&tvec);CHKERRQ(ierr);
}
}
if (n != 1 && !mglevels[n-1]->r) {
/* PCMGSetR() on the finest level if user did not supply it */
Vec *vec;
ierr = KSPCreateVecs(mglevels[n-1]->smoothd,1,&vec,0,NULL);CHKERRQ(ierr);
ierr = PCMGSetR(pc,n-1,*vec);CHKERRQ(ierr);
ierr = VecDestroy(vec);CHKERRQ(ierr);
ierr = PetscFree(vec);CHKERRQ(ierr);
}
}
if (pc->dm) {
/* need to tell all the coarser levels to rebuild the matrix using the DM for that level */
for (i=0; i<n-1; i++) {
if (mglevels[i]->smoothd->setupstage != KSP_SETUP_NEW) mglevels[i]->smoothd->setupstage = KSP_SETUP_NEWMATRIX;
}
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu == mglevels[i]->smoothd || mg->am == PC_MG_FULL || mg->am == PC_MG_KASKADE || mg->cyclesperpcapply > 1){
/* if doing only down then initial guess is zero */
ierr = KSPSetInitialGuessNonzero(mglevels[i]->smoothd,PETSC_TRUE);CHKERRQ(ierr);
}
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[i]->smoothd);CHKERRQ(ierr);
if (mglevels[i]->smoothd->reason == KSP_DIVERGED_PCSETUP_FAILED) {
pc->failedreason = PC_SUBPC_ERROR;
}
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
if (!mglevels[i]->residual) {
Mat mat;
ierr = KSPGetOperators(mglevels[i]->smoothd,NULL,&mat);CHKERRQ(ierr);
ierr = PCMGSetResidual(pc,i,PCMGResidualDefault,mat);CHKERRQ(ierr);
}
}
for (i=1; i<n; i++) {
if (mglevels[i]->smoothu && mglevels[i]->smoothu != mglevels[i]->smoothd) {
Mat downmat,downpmat;
/* check if operators have been set for up, if not use down operators to set them */
ierr = KSPGetOperatorsSet(mglevels[i]->smoothu,&opsset,NULL);CHKERRQ(ierr);
if (!opsset) {
ierr = KSPGetOperators(mglevels[i]->smoothd,&downmat,&downpmat);CHKERRQ(ierr);
ierr = KSPSetOperators(mglevels[i]->smoothu,downmat,downpmat);CHKERRQ(ierr);
}
ierr = KSPSetInitialGuessNonzero(mglevels[i]->smoothu,PETSC_TRUE);CHKERRQ(ierr);
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[i]->smoothu);CHKERRQ(ierr);
if (mglevels[i]->smoothu->reason == KSP_DIVERGED_PCSETUP_FAILED) {
pc->failedreason = PC_SUBPC_ERROR;
}
if (mglevels[i]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[i]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
}
}
if (mglevels[0]->eventsmoothsetup) {ierr = PetscLogEventBegin(mglevels[0]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
ierr = KSPSetUp(mglevels[0]->smoothd);CHKERRQ(ierr);
if (mglevels[0]->smoothd->reason == KSP_DIVERGED_PCSETUP_FAILED) {
pc->failedreason = PC_SUBPC_ERROR;
}
if (mglevels[0]->eventsmoothsetup) {ierr = PetscLogEventEnd(mglevels[0]->eventsmoothsetup,0,0,0,0);CHKERRQ(ierr);}
/*
Dump the interpolation/restriction matrices plus the
Jacobian/stiffness on each level. This allows MATLAB users to
easily check if the Galerkin condition A_c = R A_f R^T is satisfied.
Only support one or the other at the same time.
*/
#if defined(PETSC_USE_SOCKET_VIEWER)
ierr = PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_mg_dump_matlab",&dump,NULL);CHKERRQ(ierr);
if (dump) viewer = PETSC_VIEWER_SOCKET_(PetscObjectComm((PetscObject)pc));
dump = PETSC_FALSE;
#endif
ierr = PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_mg_dump_binary",&dump,NULL);CHKERRQ(ierr);
if (dump) viewer = PETSC_VIEWER_BINARY_(PetscObjectComm((PetscObject)pc));
if (viewer) {
for (i=1; i<n; i++) {
ierr = MatView(mglevels[i]->restrct,viewer);CHKERRQ(ierr);
}
for (i=0; i<n; i++) {
ierr = KSPGetPC(mglevels[i]->smoothd,&pc);CHKERRQ(ierr);
ierr = MatView(pc->mat,viewer);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
void PETSC_STDCALL kspgetdm_(KSP ksp,DM *dm, int *__ierr ){
*__ierr = KSPGetDM(
(KSP)PetscToPointer((ksp) ),dm);
}
