/*
Given a DA generates a VecScatter context that will deliver a slice
of the global vector to each processor. In this example, each processor
receives the values i=*, j=*, k=rank, i.e. one z plane.
Note: This code is written assuming only one degree of freedom per node.
For multiple degrees of freedom per node use ISCreateBlock()
instead of ISCreateGeneral().
*/
PetscErrorCode GenerateSliceScatter(DA da,VecScatter *scatter,Vec *vslice)
{
AO ao;
PetscInt M,N,P,nslice,*sliceindices,count,i,j;
PetscMPIInt rank;
PetscErrorCode ierr;
MPI_Comm comm;
Vec vglobal;
IS isfrom,isto;
ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = DAGetAO(da,&ao);CHKERRQ(ierr);
ierr = DAGetInfo(da,0,&M,&N,&P,0,0,0,0,0,0,0);CHKERRQ(ierr);
/*
nslice is number of degrees of freedom in this processors slice
if there are more processors then z plans the extra processors get 0
elements in their slice.
*/
if (rank < P) {nslice = M*N;} else nslice = 0;
/*
Generate the local vector to hold this processors slice
*/
ierr = VecCreateSeq(PETSC_COMM_SELF,nslice,vslice);CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&vglobal);CHKERRQ(ierr);
/*
Generate the indices for the slice in the "natural" global ordering
Note: this is just an example, one could select any subset of nodes
on each processor. Just list them in the global natural ordering.
*/
ierr = PetscMalloc((nslice+1)*sizeof(PetscInt),&sliceindices);CHKERRQ(ierr);
count = 0;
if (rank < P) {
for (j=0; j<N; j++) {
for (i=0; i<M; i++) {
sliceindices[count++] = rank*M*N + j*M + i;
}
}
}
/*
Convert the indices to the "PETSc" global ordering
*/
ierr = AOApplicationToPetsc(ao,nslice,sliceindices);CHKERRQ(ierr);
/* Create the "from" and "to" index set */
/* This is to scatter from the global vector */
ierr = ISCreateGeneral(PETSC_COMM_SELF,nslice,sliceindices,&isfrom);CHKERRQ(ierr);
/* This is to gather into the local vector */
ierr = ISCreateStride(PETSC_COMM_SELF,nslice,0,1,&isto);CHKERRQ(ierr);
ierr = VecScatterCreate(vglobal,isfrom,*vslice,isto,scatter);CHKERRQ(ierr);
ierr = ISDestroy(isfrom);CHKERRQ(ierr);
ierr = ISDestroy(isto);CHKERRQ(ierr);
ierr = PetscFree(sliceindices);CHKERRQ(ierr);
return 0;
}
EXTERN_C_END
#undef __FUNCT__
#define __FUNCT__ "DAMapCoordsToPeriodicDomain"
/* -----------------------------------------------------------------------------
Checks for periodicity of a DA and Maps points outside of a domain back onto the domain
using appropriate periodicity. At the moment assumes only a 2-D DA.
----------------------------------------------------------------------------------------*/
PetscErrorCode DAMapCoordsToPeriodicDomain(DA da, PetscScalar *x, PetscScalar *y)
{
DAPeriodicType periodic_type;
PetscInt dim, gx, gy;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DAGetInfo(da, &dim, &gx, &gy, 0, 0, 0, 0, 0, 0, &periodic_type, 0);
if ( periodic_type == DA_NONPERIODIC ) {
ierr = 0;
} else {
if (periodic_type==DA_XPERIODIC || periodic_type==DA_XYPERIODIC) {
while (*x >= ( PetscScalar ) gx ) { *x -= ( PetscScalar ) gx; }
while (*x < 0.0 ) { *x += ( PetscScalar ) gx; }
}
if (periodic_type==DA_YPERIODIC || periodic_type==DA_XYPERIODIC) {
while (*y >= ( PetscScalar ) gy ) { *y -= ( PetscScalar ) gy; }
while (*y < 0.0 ) { *y += ( PetscScalar ) gy; }
}
}
PetscFunctionReturn(ierr);
}
int main(int argc,char **argv)
{
DMMG *dmmg; /* multilevel grid structure */
AppCtx user; /* user-defined work context */
PetscInt mx,my,its;
PetscErrorCode ierr;
MPI_Comm comm;
SNES snes;
DA da2;
PetscInitialize(&argc,&argv,(char *)0,help);
comm = PETSC_COMM_WORLD;
/* Problem parameters (velocity of lid, prandtl, and grashof numbers) */
ierr = PetscOptionsGetReal(PETSC_NULL,"-lidvelocity",&user.lidvelocity,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(PETSC_NULL,"-prandtl",&user.prandtl,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(PETSC_NULL,"-grashof",&user.grashof,PETSC_NULL);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create user context, set problem data, create vector data structures.
Also, compute the initial guess.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Setup Physics 2:
- Lap(T) + PR*Div([U*T,V*T]) = 0
where U and V are given by the given x.u and x.v
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DACreate2d(comm,DA_NONPERIODIC,DA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0,&da2);CHKERRQ(ierr);
ierr = DASetFieldName(da2,0,"temperature");CHKERRQ(ierr);
/* Create the solver object and attach the grid/physics info */
ierr = DMMGCreate(comm,1,&user,&dmmg);CHKERRQ(ierr);
ierr = DMMGSetDM(dmmg,(DM)da2);CHKERRQ(ierr);
ierr = DMMGSetISColoringType(dmmg,IS_COLORING_GLOBAL);CHKERRQ(ierr);
ierr = DMMGSetInitialGuess(dmmg,FormInitialGuess);CHKERRQ(ierr);
ierr = DMMGSetSNES(dmmg,FormFunction,0);CHKERRQ(ierr);
ierr = DMMGSetFromOptions(dmmg);CHKERRQ(ierr);
ierr = DAGetInfo(da2,PETSC_NULL,&mx,&my,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
user.lidvelocity = 1.0/(mx*my);
user.prandtl = 1.0;
user.grashof = 1.0;
/* Solve the nonlinear system */
ierr = DMMGSolve(dmmg);CHKERRQ(ierr);
snes = DMMGGetSNES(dmmg);
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Physics 2: Number of Newton iterations = %D\n\n", its);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free spaces
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DADestroy(da2);CHKERRQ(ierr);
ierr = DMMGDestroy(dmmg);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
PetscErrorCode ComputeRHS(DA da,Vec b)
{
PetscErrorCode ierr;
PetscInt mx,my,mz;
PetscScalar h;
PetscFunctionBegin;
ierr = DAGetInfo(da,0,&mx,&my,&mz,0,0,0,0,0,0,0);CHKERRQ(ierr);
h = 1.0/((mx-1)*(my-1)*(mz-1));
ierr = VecSet(b,h);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETSC_STDCALL dagetownershipranges_(DA *da,PetscInt lx[],PetscInt ly[],PetscInt lz[],PetscErrorCode *ierr)
{
const PetscInt *gx,*gy,*gz;
PetscInt M,N,P,i;
CHKFORTRANNULLINTEGER(lx);
CHKFORTRANNULLINTEGER(ly);
CHKFORTRANNULLINTEGER(lz);
*ierr = DAGetInfo(*da,0,0,0,0,&M,&N,&P,0,0,0,0);if (*ierr) return;
*ierr = DAGetOwnershipRanges(*da,&gx,&gy,&gz);if (*ierr) return;
if (lx) {for (i=0; i<M; i++) {lx[i] = gx[i];}}
if (ly) {for (i=0; i<N; i++) {ly[i] = gy[i];}}
if (lz) {for (i=0; i<P; i++) {lz[i] = gz[i];}}
}
PetscErrorCode ComputeFunction(SNES snes,Vec x,Vec f,void *ctx) {
PetscInt i,Mx,xs,xm; PetscScalar *xx,*ff,hx; DA da = (DA) ctx; Vec xlocal;
DAGetInfo(da,PETSC_IGNORE,&Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
hx = 1.0/(PetscReal)(Mx-1);
DAGetLocalVector(da,&xlocal);DAGlobalToLocalBegin(da,x,INSERT_VALUES,xlocal); DAGlobalToLocalEnd(da,x,INSERT_VALUES,xlocal);
DAVecGetArray(da,xlocal,&xx); DAVecGetArray(da,f,&ff);
DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);
for (i=xs; i<xs+xm; i++) {
if (i == 0 || i == Mx-1) ff[i] = xx[i]/hx;
else ff[i] = (2.0*xx[i] - xx[i-1] - xx[i+1])/hx - hx*PetscExpScalar(xx[i]);
}
DAVecRestoreArray(da,xlocal,&xx); DARestoreLocalVector(da,&xlocal);DAVecRestoreArray(da,f,&ff);
return 0;}
/*
FormInitialGuessLocal - Forms initial approximation for this process
Input Parameters:
user - user-defined application context
X - vector (DA local vector)
Output Parameter:
X - vector with the local values set
*/
PetscErrorCode FormInitialGuessLocal(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
PetscReal grashof,dx;
Field **x;
grashof = user->grashof;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
dx = 1.0/(mx-1);
/*
Get local grid boundaries (for 2-dimensional DA):
xs, ys - starting grid indices (no ghost points)
xm, ym - widths of local grid (no ghost points)
*/
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
/*
Get a pointer to vector data.
- For default PETSc vectors, VecGetArray() returns a pointer to
the data array. Otherwise, the routine is implementation dependent.
- You MUST call VecResetArraystoreArray() when you no longer need access to
the array.
*/
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
/*
Compute initial guess over the locally owned part of the grid
Initial condition is motionless fluid and equilibrium temperature
U = V = Omega = 0.0; Temp[x_i, y_j] = x_i;
*/
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].u = 0.0;
x[j][i].v = 0.0;
x[j][i].omega = 0.0;
x[j][i].temp = (grashof>0)*i*dx;
}
}
/* Restore vector */
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
void PETSC_STDCALL dasetlocalfunction_(DA *da,void (PETSC_STDCALL *func)(DALocalInfo*,void*,void*,void*,PetscErrorCode*),PetscErrorCode *ierr)
{
PetscInt dim;
PetscObjectAllocateFortranPointers(*da,6);
*ierr = DAGetInfo(*da,&dim,0,0,0,0,0,0,0,0,0,0); if (*ierr) return;
if (dim == 2) {
((PetscObject)*da)->fortran_func_pointers[4] = (PetscVoidFunction)func;
*ierr = DASetLocalFunction(*da,(DALocalFunction1)ourlf2d);
} else if (dim == 3) {
((PetscObject)*da)->fortran_func_pointers[5] = (PetscVoidFunction)func;
*ierr = DASetLocalFunction(*da,(DALocalFunction1)ourlf3d);
} else if (dim == 1) {
((PetscObject)*da)->fortran_func_pointers[3] = (PetscVoidFunction)func;
*ierr = DASetLocalFunction(*da,(DALocalFunction1)ourlf1d);
} else *ierr = 1;
}
void PETSC_STDCALL dasetlocaljacobian_(DA *da,void (PETSC_STDCALL *jac)(DALocalInfo*,void*,void*,void*,PetscErrorCode*),PetscErrorCode *ierr)
{
PetscInt dim;
PetscObjectAllocateFortranPointers(*da,6);
*ierr = DAGetInfo(*da,&dim,0,0,0,0,0,0,0,0,0,0); if (*ierr) return;
if (dim == 2) {
((PetscObject)*da)->fortran_func_pointers[1] = (PetscVoidFunction)jac;
*ierr = DASetLocalJacobian(*da,(DALocalFunction1)ourlj2d);
} else if (dim == 3) {
((PetscObject)*da)->fortran_func_pointers[2] = (PetscVoidFunction)jac;
*ierr = DASetLocalJacobian(*da,(DALocalFunction1)ourlj3d);
} else if (dim == 1) {
((PetscObject)*da)->fortran_func_pointers[0] = (PetscVoidFunction)jac;
*ierr = DASetLocalJacobian(*da,(DALocalFunction1)ourlj1d);
} else *ierr = 1;
}
PetscErrorCode ComputeJacobian(SNES snes,Vec x,Mat *J,Mat *B,MatStructure *flag,void *ctx){
DA da = (DA) ctx; PetscInt i,Mx,xm,xs; PetscScalar hx,*xx; Vec xlocal;
DAGetInfo(da,PETSC_IGNORE,&Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
hx = 1.0/(PetscReal)(Mx-1);
DAGetLocalVector(da,&xlocal);DAGlobalToLocalBegin(da,x,INSERT_VALUES,xlocal); DAGlobalToLocalEnd(da,x,INSERT_VALUES,xlocal);
DAVecGetArray(da,xlocal,&xx);
DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);
for (i=xs; i<xs+xm; i++) {
if (i == 0 || i == Mx-1) { MatSetValue(*J,i,i,1.0/hx,INSERT_VALUES);}
else {
MatSetValue(*J,i,i-1,-1.0/hx,INSERT_VALUES);
MatSetValue(*J,i,i,2.0/hx - hx*PetscExpScalar(xx[i]),INSERT_VALUES);
MatSetValue(*J,i,i+1,-1.0/hx,INSERT_VALUES);
}
}
MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY); MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY); *flag = SAME_NONZERO_PATTERN;
DAVecRestoreArray(da,xlocal,&xx);DARestoreLocalVector(da,&xlocal);
return 0;}
PetscErrorCode CharacteristicSetUp_DA(Characteristic c)
{
PetscMPIInt blockLen[2];
MPI_Aint indices[2];
MPI_Datatype oldtypes[2];
PetscInt dim, numValues;
PetscErrorCode ierr;
ierr = DAGetInfo(c->velocityDA, &dim, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);CHKERRQ(ierr);
if (c->structured) {
c->numIds = dim;
} else {
c->numIds = 3;
}
if (c->numFieldComp > MAX_COMPONENTS) {
SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE, "The maximum number of fields allowed is %d, you have %d. You can recompile after increasing MAX_COMPONENTS.", MAX_COMPONENTS, c->numFieldComp);
}
numValues = 4 + MAX_COMPONENTS;
/* Create new MPI datatype for communication of characteristic point structs */
blockLen[0] = 1+c->numIds; indices[0] = 0; oldtypes[0] = MPIU_INT;
blockLen[1] = numValues; indices[1] = (1+c->numIds)*sizeof(PetscInt); oldtypes[1] = MPIU_SCALAR;
ierr = MPI_Type_struct(2, blockLen, indices, oldtypes, &c->itemType);CHKERRQ(ierr);
ierr = MPI_Type_commit(&c->itemType);CHKERRQ(ierr);
/* Initialize the local queue for char foot values */
ierr = VecGetLocalSize(c->velocity, &c->queueMax);CHKERRQ(ierr);
ierr = PetscMalloc(c->queueMax * sizeof(CharacteristicPointDA2D), &c->queue);CHKERRQ(ierr);
c->queueSize = 0;
/* Allocate communication structures */
if (c->numNeighbors <= 0) {
SETERRQ1(PETSC_ERR_ARG_WRONGSTATE, "Invalid number of neighbors %d. Call CharactersiticSetNeighbors() before setup.", c->numNeighbors);
}
ierr = PetscMalloc(c->numNeighbors * sizeof(PetscInt), &c->needCount);CHKERRQ(ierr);
ierr = PetscMalloc(c->numNeighbors * sizeof(PetscInt), &c->localOffsets);CHKERRQ(ierr);
ierr = PetscMalloc(c->numNeighbors * sizeof(PetscInt), &c->fillCount);CHKERRQ(ierr);
ierr = PetscMalloc(c->numNeighbors * sizeof(PetscInt), &c->remoteOffsets);CHKERRQ(ierr);
ierr = PetscMalloc((c->numNeighbors-1) * sizeof(MPI_Request), &c->request);CHKERRQ(ierr);
ierr = PetscMalloc((c->numNeighbors-1) * sizeof(MPI_Status), &c->status);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode doit(DA da,Vec global)
{
PetscErrorCode ierr;
PetscInt i,j,k,M,N,dof;
ierr = DAGetInfo(da,0,&M,&N,0,0,0,0,&dof,0,0,0);CHKERRQ(ierr);
{
struct {PetscScalar inside[dof];} **mystruct;
ierr = DAVecGetArray(da,global,(void*) &mystruct);
for ( i=0; i<N; i++) {
for ( j=0; j<M; j++) {
for ( k=0; k<dof; k++) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"%d %d %G\n",i,j,mystruct[i][j].inside[0]);CHKERRQ(ierr);
mystruct[i][j].inside[1] = 2.1;
}
}
}
ierr = DAVecRestoreArray(da,global,(void*) &mystruct);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/* Form initial guess for Physic 1 */
PetscErrorCode FormInitialGuessLocal1(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
Field1 **x;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].u = 0.0;
x[j][i].v = 0.0;
x[j][i].omega = 0.0;
}
}
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
/*@
DAAppSetADElementFunctionGradient - Set routine that evaluates the
local part of a function on a 2-dimensional DA with 1 degree of freedom.
Collective on TAO_APPLICATION
Input Parameters:
+ daapp - the TAO_DA_APPLICATION solver context
. funcgrad - local function gradient routine
. flops - the number of flops done performed in the funcgrad routine
- fgctx - [optional] user-defined context for private data for the evaluation.
Calling sequence of funcgrad:
$ int funcgrad(int coordinates[2], PetscScalar x[4], double *f, PetscScalar g[4], void* ctx)
+ coord - the global coordinates [i j] in each direction of the DA
. x - the variables on the DA ( da[j][i], da[j][j+1], da[j+1][i], da[j+1][i+1] ) (bottom left, bottom right, top left, top right)
. g - the ADIC differentiated objective function with respect to each variable
- ctx - user defined context
Note: This function requires ADIC to be installed and the ADIC-specific variables to be set in
$TAO_DIR/bmake/packages.$PETSC_ARCH
Level: intermediate
.keywords: DA, gradient, ADIC
.seealso: DAAppSetObjectiveAndGradientRoutine(), DAAppSetElementObjectiveAndGradientRoutine()
@*/
int DAAppSetADElementFunctionGradient(TAO_APPLICATION daapplication,
int (*funcgrad)(int[2],DERIV_TYPE[4],DERIV_TYPE*,void*),
int flops, void *ctx){
int i,n,info;
int dim,dof,s;
DAStencilType st;
TaoDA2D1DOFADICFGCtx *fgctx;
DA da;
PetscFunctionBegin;
info=DAAppGetNumberOfDAGrids(daapplication,&n); CHKERRQ(info);
for (i=0;i<n;i++){
info = DAAppGetDA(daapplication, i, &da); CHKERRQ(info);
info = DAGetInfo(da,&dim,0,0,0,0,0,0,&dof,&s,0,&st); CHKERRQ(info);
if (dim!=2){
SETERRQ(1,"TAO DA ERROR: DA must have dimension of 2");}
if (dof!=1){
SETERRQ(1,"TAO DA ERROR: DA must have exactly 1 degree of freedom per node");}
if (s!=1){
SETERRQ(1,"TAO DA ERROR: DA stencil width must equal 1"); }
if (st!=DA_STENCIL_BOX){
SETERRQ(1,"TAO DA ERROR: DA stencil must be DA_STENCIL_BOX");}
}
PetscNew(TaoDA2D1DOFADICFGCtx,&fgctx);
// ad_AD_Init(4);
ad_AD_Init(ad_GRAD_MAX);
ad_AD_SetIndepArray(fgctx->adX,4);
ad_AD_SetIndepDone();
fgctx->computeadicfunctiongradient=funcgrad;
fgctx->elementfgctx=ctx;
fgctx->elementfgflops=flops;
info = DAAppSetObjectiveAndGradientRoutine(daapplication, TaoDA2dLoopADFunctionGradient, (void*)fgctx);
CHKERRQ(info);
info = TaoAppSetDestroyRoutine(daapplication,TaoApplicationFreeMemory, (void*)fgctx); CHKERRQ(info);
info = TaoAppSetDestroyRoutine(daapplication, TaoShutDownADICQQQQ, 0); CHKERRQ(info);
info = PetscInfo(daapplication,"Set objective function pointer for TAO_DA_APPLICATION object.\n"); CHKERRQ(info);
PetscFunctionReturn(0);
}
static int MyGridMonitorBefore(TAO_APPLICATION myapp, DA da, PetscInt level, void *ctx) {
AppCtx *user = (AppCtx*)ctx;
int info;
PetscInt mx,my;
info = DAGetInfo(da,PETSC_NULL,&mx,&my,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,
PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(info);
user->mx = mx;
user->my = my;
user->hx = (user->r - user->l) / (user->mx - 1);
user->hy = (user->t - user->b) / (user->my - 1);
user->area = 0.5 * user->hx * user->hy;
user->fgctx.hx = user->hx;
user->fgctx.hy = user->hy;
user->fgctx.area = user->area;
user->bmx=(PetscInt)((mx+1)*user->fx); user->bmy=(PetscInt)((my+1)*user->fy);
PetscPrintf(MPI_COMM_WORLD,"Grid: %d, mx: %d my: %d \n",level,mx,my);
return 0;
}
/* Form initial guess for Physic 2 */
PetscErrorCode FormInitialGuessLocal2(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
PetscReal grashof,dx;
Field2 **x;
grashof = user->grashof;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
dx = 1.0/(mx-1);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].temp = (grashof>0)*i*dx;
}
}
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
extern PetscErrorCode DefiantComputeTransmissibilities(
BlackOilReservoirSimulation* MySim) {
PetscErrorCode ierr;
PetscInt i, j, k, mx, my, mz, xm, ym, zm, xs, ys, zs;
PetscScalar ***LocalFlowMask;
/* Area * Permeability divided by height */
PetscScalar ***LocalAKHx1m, ***LocalAKHx2m, ***LocalAKHx3m;
PetscScalar ***LocalAKHx1p, ***LocalAKHx2p, ***LocalAKHx3p;
/* Local viscosity at the faces */
PetscScalar ***LocalMuox1m, ***LocalMuox1p, ***LocalMuwx1m, ***LocalMuwx1p,
***LocalMugx1m, ***LocalMugx1p;
PetscScalar ***LocalMuox2m, ***LocalMuox2p, ***LocalMuwx2m, ***LocalMuwx2p,
***LocalMugx2m, ***LocalMugx2p;
PetscScalar ***LocalMuox3m, ***LocalMuox3p, ***LocalMuwx3m, ***LocalMuwx3p,
***LocalMugx3m, ***LocalMugx3p;
/* Relative Permeabilities at the faces */
PetscScalar ***LocalRelPermox1m, ***LocalRelPermox1p, ***LocalRelPermox2m,
***LocalRelPermox2p, ***LocalRelPermox3m, ***LocalRelPermox3p;
PetscScalar ***LocalRelPermwx1m, ***LocalRelPermwx1p, ***LocalRelPermwx2m,
***LocalRelPermwx2p, ***LocalRelPermwx3m, ***LocalRelPermwx3p;
PetscScalar ***LocalRelPermgx1m, ***LocalRelPermgx1p, ***LocalRelPermgx2m,
***LocalRelPermgx2p, ***LocalRelPermgx3m, ***LocalRelPermgx3p;
/* Volume factors at cell center */
PetscScalar ***LocalBo, ***LocalBw, ***LocalBg;
/* Volume factors at cell faces */
PetscScalar ***LocalBox1p, ***LocalBox2p, ***LocalBox3p;
PetscScalar ***LocalBox1m, ***LocalBox2m, ***LocalBox3m;
PetscScalar ***LocalBwx1p, ***LocalBwx2p, ***LocalBwx3p;
PetscScalar ***LocalBwx1m, ***LocalBwx2m, ***LocalBwx3m;
PetscScalar ***LocalBgx1p, ***LocalBgx2p, ***LocalBgx3p;
PetscScalar ***LocalBgx1m, ***LocalBgx2m, ***LocalBgx3m;
/* Transmissibility For Oil*/
PetscScalar ***LocalTox1p, ***LocalTox2p, ***LocalTox3p;
PetscScalar ***LocalTox1m, ***LocalTox2m, ***LocalTox3m;
/* Transmissibility For Water*/
PetscScalar ***LocalTwx1p, ***LocalTwx2p, ***LocalTwx3p;
PetscScalar ***LocalTwx1m, ***LocalTwx2m, ***LocalTwx3m;
/* Transmissibility For Gas*/
PetscScalar ***LocalTgx1p, ***LocalTgx2p, ***LocalTgx3p;
PetscScalar ***LocalTgx1m, ***LocalTgx2m, ***LocalTgx3m;
PetscFunctionBegin;
/* Get dimensions and extents of the local vectors */
ierr = DAGetInfo(MySim->SimDA, 0, &mx, &my, &mz, 0, 0, 0, 0, 0, 0, 0);CHKERRQ(ierr);
ierr = DAGetCorners(MySim->SimDA, &xs, &ys, &zs, &xm, &ym, &zm);CHKERRQ(ierr);
/* Grab the data for the flow field */
ierr = DAVecGetArray(MySim->SimDA, MySim->FlowMask, &LocalFlowMask);CHKERRQ(ierr);
/* Grab the local data for area * permeability divided by height */
ierr = DAVecGetArray(MySim->SimDA, MySim->AKHx1m, &LocalAKHx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->AKHx2m, &LocalAKHx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->AKHx3m, &LocalAKHx3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->AKHx1p, &LocalAKHx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->AKHx2p, &LocalAKHx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->AKHx3p, &LocalAKHx3p);CHKERRQ(ierr);
/* Grab the local data for the face viscosities */
ierr = DAVecGetArray(MySim->SimDA, MySim->Muox1m, &LocalMuox1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muox1p, &LocalMuox1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muwx1m, &LocalMuwx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muwx1p, &LocalMuwx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Mugx1m, &LocalMugx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Mugx1p, &LocalMugx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muox2m, &LocalMuox2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muox2p, &LocalMuox2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muwx2m, &LocalMuwx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muwx2p, &LocalMuwx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Mugx2m, &LocalMugx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Mugx2p, &LocalMugx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muox3m, &LocalMuox3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muox3p, &LocalMuox3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muwx3m, &LocalMuwx3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Muwx3p, &LocalMuwx3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Mugx3m, &LocalMugx3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Mugx3p, &LocalMugx3p);CHKERRQ(ierr);
/* Grab the local data for the face Relative Permeabilities */
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermox1m, &LocalRelPermox1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermox1p, &LocalRelPermox1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermox2m, &LocalRelPermox2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermox2p, &LocalRelPermox2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermox3m, &LocalRelPermox3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermox3p, &LocalRelPermox3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermwx1m, &LocalRelPermwx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermwx1p, &LocalRelPermwx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermwx2m, &LocalRelPermwx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermwx2p, &LocalRelPermwx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermwx3m, &LocalRelPermwx3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermwx3p, &LocalRelPermwx3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermgx1m, &LocalRelPermgx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermgx1p, &LocalRelPermgx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermgx2m, &LocalRelPermgx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermgx2p, &LocalRelPermgx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermgx3m, &LocalRelPermgx3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->RelPermgx3p, &LocalRelPermgx3p);CHKERRQ(ierr);
/* Grab the local data for volume factors at the cell centers */
ierr = DAVecGetArray(MySim->SimDA, MySim->Bo, &LocalBo);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bw, &LocalBw);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bg, &LocalBg);CHKERRQ(ierr);
/* Grab the local data for Volume factors at the faces */
ierr = DAVecGetArray(MySim->SimDA, MySim->Box1p, &LocalBox1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Box2p, &LocalBox2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Box3p, &LocalBox3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Box1m, &LocalBox1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Box2m, &LocalBox2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Box3m, &LocalBox3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bwx1p, &LocalBwx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bwx2p, &LocalBwx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bwx3p, &LocalBwx3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bwx1m, &LocalBwx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bwx2m, &LocalBwx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bwx3m, &LocalBwx3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bgx1p, &LocalBgx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bgx2p, &LocalBgx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bgx3p, &LocalBgx3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bgx1m, &LocalBgx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bgx2m, &LocalBgx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Bgx3m, &LocalBgx3m);CHKERRQ(ierr);
/* Grab the local data for the transmissibilities */
ierr = DAVecGetArray(MySim->SimDA, MySim->Tox1m, &LocalTox1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tox1p, &LocalTox1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Twx1m, &LocalTwx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Twx1p, &LocalTwx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tgx1m, &LocalTgx1m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tgx1p, &LocalTgx1p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tox2m, &LocalTox2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tox2p, &LocalTox2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Twx2m, &LocalTwx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Twx2p, &LocalTwx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tgx2m, &LocalTgx2m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tgx2p, &LocalTgx2p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tox3m, &LocalTox3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tox3p, &LocalTox3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Twx3m, &LocalTwx3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Twx3p, &LocalTwx3p);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tgx3m, &LocalTgx3m);CHKERRQ(ierr);
ierr = DAVecGetArray(MySim->SimDA, MySim->Tgx3p, &LocalTgx3p);CHKERRQ(ierr);
for (k = zs; k < zs + zm; k++) {
for (j = ys; j < ys + ym; j++) {
for (i = xs; i < xs + xm; i++) {
if (i == 0 || j == 0 || k == 0 || i == mx - 1 || j == my - 1 || k == mz
- 1) {
} else if (ABS(LocalFlowMask[k][j][i]-FLUID_FLOW) < EPSILON) {
/* Transmissibility */
LocalTox1m[k][j][i] = LocalRelPermox1m[k][j][i]
* LocalAKHx1m[k][j][i] / LocalMuox1m[k][j][i] / LocalBox1m[k][j][i];
LocalTox1p[k][j][i] = LocalRelPermox1p[k][j][i]
* LocalAKHx1p[k][j][i] / LocalMuox1p[k][j][i] / LocalBox1p[k][j][i];
LocalTox2m[k][j][i] = LocalRelPermox2m[k][j][i]
* LocalAKHx2m[k][j][i] / LocalMuox2m[k][j][i] / LocalBox2m[k][j][i];
LocalTox2p[k][j][i] = LocalRelPermox2p[k][j][i]
* LocalAKHx2p[k][j][i] / LocalMuox2p[k][j][i] / LocalBox2p[k][j][i];
LocalTox3m[k][j][i] = LocalRelPermox3m[k][j][i]
* LocalAKHx3m[k][j][i] / LocalMuox3m[k][j][i] / LocalBox3m[k][j][i];
LocalTox3p[k][j][i] = LocalRelPermox3p[k][j][i]
* LocalAKHx3p[k][j][i] / LocalMuox3p[k][j][i] / LocalBox3p[k][j][i];
LocalTwx1m[k][j][i] = LocalRelPermwx1m[k][j][i]
* LocalAKHx1m[k][j][i] / LocalMuwx1m[k][j][i] / LocalBwx1m[k][j][i];
LocalTwx1p[k][j][i] = LocalRelPermwx1p[k][j][i]
* LocalAKHx1p[k][j][i] / LocalMuwx1p[k][j][i] / LocalBwx1p[k][j][i];
LocalTwx2m[k][j][i] = LocalRelPermwx2m[k][j][i]
* LocalAKHx2m[k][j][i] / LocalMuwx2m[k][j][i] / LocalBwx2m[k][j][i];
LocalTwx2p[k][j][i] = LocalRelPermwx2p[k][j][i]
* LocalAKHx2p[k][j][i] / LocalMuwx2p[k][j][i] / LocalBwx2p[k][j][i];
LocalTwx3m[k][j][i] = LocalRelPermwx3m[k][j][i]
* LocalAKHx3m[k][j][i] / LocalMuwx3m[k][j][i] / LocalBwx3m[k][j][i];
LocalTwx3p[k][j][i] = LocalRelPermwx3p[k][j][i]
* LocalAKHx3p[k][j][i] / LocalMuwx3p[k][j][i] / LocalBwx3p[k][j][i];
LocalTgx1m[k][j][i] = LocalRelPermgx1m[k][j][i]
* LocalAKHx1m[k][j][i] / LocalMugx1m[k][j][i] / LocalBgx1m[k][j][i];
LocalTgx1p[k][j][i] = LocalRelPermgx1p[k][j][i]
* LocalAKHx1p[k][j][i] / LocalMugx1p[k][j][i] / LocalBgx1p[k][j][i];
LocalTgx2m[k][j][i] = LocalRelPermgx2m[k][j][i]
* LocalAKHx2m[k][j][i] / LocalMugx2m[k][j][i] / LocalBgx2m[k][j][i];
LocalTgx2p[k][j][i] = LocalRelPermgx2p[k][j][i]
* LocalAKHx2p[k][j][i] / LocalMugx2p[k][j][i] / LocalBgx2p[k][j][i];
LocalTgx3m[k][j][i] = LocalRelPermgx3m[k][j][i]
* LocalAKHx3m[k][j][i] / LocalMugx3m[k][j][i] / LocalBgx3m[k][j][i];
LocalTgx3p[k][j][i] = LocalRelPermgx3p[k][j][i]
* LocalAKHx3p[k][j][i] / LocalMugx3p[k][j][i] / LocalBgx3p[k][j][i];
}
}
}
}
/* Restore the new arrays to their reightful place */
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tox1m, &LocalTox1m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tox1p, &LocalTox1p);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Twx1m, &LocalTwx1m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Twx1p, &LocalTwx1p);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tgx1m, &LocalTgx1m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tgx1p, &LocalTgx1p);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tox2m, &LocalTox2m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tox2p, &LocalTox2p);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Twx2m, &LocalTwx2m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Twx2p, &LocalTwx2p);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tgx2m, &LocalTgx2m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tgx2p, &LocalTgx2p);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tox3m, &LocalTox3m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tox3p, &LocalTox3p);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Twx3m, &LocalTwx3m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Twx3p, &LocalTwx3p);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tgx3m, &LocalTgx3m);CHKERRQ(ierr);
ierr = DAVecRestoreArray(MySim->SimDA, MySim->Tgx3p, &LocalTgx3p);CHKERRQ(ierr);
/* Begin Assembly for vectors */
ierr = VecAssemblyBegin(MySim->Tox1p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tox2p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tox3p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tox1m);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tox2m);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tox3m);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Twx1p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Twx2p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Twx3p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Twx1m);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Twx2m);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Twx3m);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tgx1p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tgx2p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tgx3p);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tgx1m);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tgx2m);CHKERRQ(ierr);
ierr = VecAssemblyBegin(MySim->Tgx3m);CHKERRQ(ierr);
/* And end Assembly */
ierr = VecAssemblyEnd(MySim->Tox1p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tox2p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tox3p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tox1m);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tox2m);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tox3m);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Twx1p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Twx2p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Twx3p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Twx1m);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Twx2m);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Twx3m);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tgx1p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tgx2p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tgx3p);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tgx1m);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tgx2m);CHKERRQ(ierr);
ierr = VecAssemblyEnd(MySim->Tgx3m);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void getForces(Vec params, std::vector<Vec> &forces, DA da, timeInfo ti, int numParams) {
#ifdef __DEBUG__
std::cout << RED"Entering "NRM << __func__ << std::endl;
#endif
// Generate the force vector based on the current parameters ...
// F = Sum { B_i a_i}
PetscScalar * pVec;
VecGetArray(params, &pVec);
// Clear the Forces
for (unsigned int i=0; i<forces.size(); i++) {
if (forces[i] != NULL) {
VecDestroy(forces[i]);
}
}
forces.clear();
unsigned int numSteps = (unsigned int)(ceil(( ti.stop - ti.start)/ti.step));
// create and initialize to 0
for (unsigned int i=0; i<numSteps+1; i++) {
Vec tmp;
DACreateGlobalVector(da, &tmp);
VecZeroEntries(tmp);
forces.push_back(tmp);
}
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
PetscScalar ***tauArray;
unsigned int paramTimeSteps = (unsigned int)(ceil(( (double)(numSteps))/ ((double)(2*numParams)) ));
double acx,acy,acz;
int x, y, z, m, n, p;
int mx,my,mz;
DAGetCorners(da, &x, &y, &z, &m, &n, &p);
DAGetInfo(da,0, &mx, &my, &mz, 0,0,0,0,0,0,0);
double hx = 1.0/(mx-1.0);
for (int b=0; b<numParams; b++) {
std::vector<Vec> tau;
unsigned int tBegin = paramTimeSteps*b;
unsigned int tEnd = tBegin + numSteps/2; // paramTimeSteps*(b+2);
// std::cout << "For param " << b << ": Time step range is " << tBegin << " -> " << tEnd << std::endl;
for (unsigned int t=0; t<numSteps+1; t++) {
double newTime = (ti.step*(t-tBegin)*numSteps)/((double)(paramTimeSteps));
if ( (t>=tBegin) && (t<=tEnd)) {
DAVecGetArray(da, forces[t], &tauArray);
for (int k = z; k < z + p ; k++) {
for (int j = y; j < y + n; j++) {
for (int i = x; i < x + m; i++) {
acx = (i)*hx; acy = (j)*hx; acz = (k)*hx;
tauArray[k][j][i] += pVec[b]*sin(M_PI*newTime)*cos(2*M_PI*acx)*cos(2*M_PI*acy)*cos(2*M_PI*acz);
}
}
}
DAVecRestoreArray ( da, forces[t], &tauArray ) ;
}
}
}
VecRestoreArray(params, &pVec);
#ifdef __DEBUG__
// Get the norms of the forces ... just to be safe ..
double fNorm1, fNorm2;
for (unsigned int i=0; i<forces.size(); i++) {
VecNorm(forces[i], NORM_INFINITY, &fNorm1);
VecNorm(forces[i], NORM_2, &fNorm2);
PetscPrintf(0, "Force Norms at timestep %d are %g and %g\n", i, fNorm1, fNorm2);
}
#endif
#ifdef __DEBUG__
std::cout << GRN"Leaving "NRM << __func__ << std::endl;
#endif
}
int main(int argc, char **argv)
{
PetscInitialize(&argc, &argv, "wave.opt", help);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
double startTime, endTime;
int Ns = 32;
unsigned int dof = 1;
// double dtratio = 1.0;
DA da; // Underlying DA
Vec rho; // density - elemental scalar
Vec nu; // Lame parameter - lambda - elemental scalar
std::vector < std::vector<Vec> > fBasis; // the scalar activation - nodal scalar
// std::vector<Vec> truth; // the ground truth.
// Initial conditions
Vec initialDisplacement;
Vec initialVelocity;
timeInfo ti;
// get Ns
CHKERRQ ( PetscOptionsGetInt(0,"-Ns", &Ns,0) );
double t0 = 0.0;
double dt = 1.0/(Ns);
double t1 = 1.0;
double nuVal = 1.0;
double beta = 0.0001;
int numParams = 5;
CHKERRQ ( PetscOptionsGetInt(0,"-nump",&numParams,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-t0",&t0,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-t1",&t1,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-dt",&dt,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-nu",&nuVal,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-beta",&beta,0) );
// CHKERRQ ( PetscOptionsGetString(PETSC_NULL, "-pn", problemName, PETSC_MAX_PATH_LEN-1, PETSC_NULL));
// Time info for timestepping
ti.start = t0;
ti.stop = t1;
ti.step = dt;
if (!rank) {
std::cout << "Problem size is " << Ns+1 << " spatially and NT = " << (int)ceil(1.0/dt) << std::endl << std::endl;
std::cout << "Number of parameters is " << numParams << std::endl;
}
// create DA
CHKERRQ ( DACreate3d ( PETSC_COMM_WORLD, DA_NONPERIODIC, DA_STENCIL_BOX,
Ns+1, Ns+1, Ns+1, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE,
1, 1, 0, 0, 0, &da) );
massMatrix *Mass = new massMatrix(feMat::PETSC); // Mass Matrix
stiffnessMatrix *Stiffness = new stiffnessMatrix(feMat::PETSC); // Stiffness matrix
waveDamping *Damping = new waveDamping(feMat::PETSC); // Damping Matrix
fdynamicVector *Force = new fdynamicVector(feVec::PETSC); // Force Vector
// create vectors
CHKERRQ( DACreateGlobalVector(da, &rho) );
CHKERRQ( DACreateGlobalVector(da, &nu) );
CHKERRQ( DACreateGlobalVector(da, &initialDisplacement) );
CHKERRQ( DACreateGlobalVector(da, &initialVelocity) );
// Set initial conditions
CHKERRQ( VecSet ( initialDisplacement, 0.0) );
CHKERRQ( VecSet ( initialVelocity, 0.0) );
VecZeroEntries( nu );
VecZeroEntries( rho );
CHKERRQ( VecSet ( nu, nuVal) );
CHKERRQ( VecSet ( rho, 1.0) );
int x, y, z, m, n, p;
int mx,my,mz;
CHKERRQ( DAGetCorners(da, &x, &y, &z, &m, &n, &p) );
CHKERRQ( DAGetInfo(da,0, &mx, &my, &mz, 0,0,0,0,0,0,0) );
double acx,acy,acz;
double hx = 1.0/((double)Ns);
// allocate for temporary buffers ...
// unsigned int elemSize = Ns*Ns*Ns;
// std::cout << "Elem size is " << elemSize << std::endl;
// unsigned int nodeSize = (Ns+1)*(Ns+1)*(Ns+1);
// Now set the activation ...
unsigned int numSteps = (unsigned int)(ceil(( ti.stop - ti.start)/ti.step));
// Vec tauVec;
// PetscScalar ***tauArray;
unsigned int paramTimeSteps = (unsigned int)(ceil(( (double)(numSteps))/ ((double)(2*numParams)) ));
/*
for (int b=0; b<numParams; b++) {
std::vector<Vec> tau;
unsigned int tBegin = paramTimeSteps*b;
unsigned int tEnd = tBegin + numSteps/2; // paramTimeSteps*(b+2);
// std::cout << "For param " << b << ": Time step range is " << tBegin << " -> " << tEnd << std::endl;
for (unsigned int t=0; t<numSteps+1; t++) {
double newTime = (dt*(t-tBegin)*numSteps)/((double)(paramTimeSteps));
// double fff = 0.0;
CHKERRQ( DACreateGlobalVector(da, &tauVec) );
CHKERRQ( VecSet( tauVec, 0.0));
if ( (t>=tBegin) && (t<=tEnd)) {
CHKERRQ(DAVecGetArray(da, tauVec, &tauArray));
for (int k = z; k < z + p ; k++) {
for (int j = y; j < y + n; j++) {
for (int i = x; i < x + m; i++) {
acx = (i)*hx; acy = (j)*hx; acz = (k)*hx;
tauArray[k][j][i] = sin(M_PI*newTime)*cos(2*M_PI*acx)*cos(2*M_PI*acy)*cos(2*M_PI*acz);
}
}
}
CHKERRQ( DAVecRestoreArray ( da, tauVec, &tauArray ) );
}
tau.push_back(tauVec);
}
fBasis.push_back(tau);
}
*/
// std::cout << "Finished setting basis" << std::endl;
/*
// Set initial velocity ...
CHKERRQ(DAVecGetArray(da, initialVelocity, &solArray));
for (int k = z; k < z + p ; k++) {
for (int j = y; j < y + n; j++) {
for (int i = x; i < x + m; i++) {
acx = (i)*hx; acy = (j)*hx; acz = (k)*hx;
solArray[k][j][i] = M_PI*cos(2*M_PI*acx)*cos(2*M_PI*acy)*cos(2*M_PI*acz);
}
}
}
CHKERRQ( DAVecRestoreArray ( da, initialVelocity, &solArray ) );
*/
std::vector<Vec> newF;
Vec alpha;
PetscScalar *avec;
VecCreateSeq(PETSC_COMM_SELF, numParams, &alpha);
/*
VecCreate(PETSC_COMM_WORLD, &alpha);
VecSetSizes(alpha, numParams, PETSC_DECIDE);
VecSetFromOptions(alpha);
*/
VecGetArray(alpha, &avec);
for (int j=0; j<numParams; j++)
avec[j] = 0.5 + 0.5*j;
VecRestoreArray(alpha, &avec);
// getForces(alpha, fBasis, newF);
getForces(alpha, newF, da, ti, numParams);
// Setup Matrices and Force Vector ...
Mass->setProblemDimensions(1.0, 1.0, 1.0);
Mass->setDA(da);
Mass->setDof(dof);
Mass->setNuVec(rho);
Stiffness->setProblemDimensions(1.0, 1.0, 1.0);
Stiffness->setDA(da);
Stiffness->setDof(dof);
Stiffness->setNuVec(nu);
Damping->setAlpha(0.0);
Damping->setBeta(0.00075);
Damping->setMassMatrix(Mass);
Damping->setStiffnessMatrix(Stiffness);
Damping->setDA(da);
Damping->setDof(dof);
// Force Vector
Force->setProblemDimensions(1.0,1.0,1.0);
Force->setDA(da);
Force->setFDynamic(newF);
Force->setTimeInfo(&ti);
// Newmark time stepper ...
newmark *ts = new newmark;
ts->setMassMatrix(Mass);
ts->setDampingMatrix(Damping);
ts->setStiffnessMatrix(Stiffness);
ts->damp(false);
ts->setTimeFrames(1);
ts->storeVec(true);
ts->setAdjoint(false);
ts->setForceVector(Force);
ts->setInitialDisplacement(initialDisplacement);
ts->setInitialVelocity(initialVelocity);
ts->setTimeInfo(&ti);
ts->setAdjoint(false); // set if adjoint or forward
ts->init(); // initialize IMPORTANT
// if (!rank)
// std::cout << RED"Starting initial forward solve"NRM << std::endl;
ts->solve();// solve
// if (!rank)
// std::cout << GRN"Finished with initial forward solve"NRM << std::endl;
std::vector<Vec> solvec = ts->getSolution();
// Now lets check the error ...
// Vec nr;
// concatenateVecs(solvec, nr);
// VecDestroy(nr);
// VecDestroy(gt);
// std::cout << std::endl;
/*************
* INVERSE *
*************/
// True solution is tau ... we want to recover it.
// The observations in this case are, solvec
/* Set very initial guess for the inverse problem*/
// Now can clear memory ...
/*
for (int i=0; i<newF.size(); i++) {
if (newF[i] != NULL) {
VecDestroy(newF[i]);
}
}
newF.clear();
for (int i=0; i<solvec.size(); i++) {
if (solvec[i] != NULL) {
VecDestroy(solvec[i]);
}
}
solvec.clear();
ts->destroy();
VecDestroy(rho);
VecDestroy(nu);
VecDestroy(initialDisplacement);
VecDestroy(initialVelocity);
VecDestroy(alpha);
DADestroy(da);
PetscFinalize();
return 0;
*/
Vec gt, nr;
concatenateVecs(solvec, gt);
Vec guess;
VecDuplicate(alpha, &guess);
VecZeroEntries(guess);
// VecDuplicate(guess, &Out);
// VecZeroEntries(Out);
// double norm;
/*
PetscRandom rctx;
PetscRandomCreate(PETSC_COMM_WORLD, &rctx);
PetscRandomSetFromOptions(rctx);
VecSetRandom(guess, rctx);
VecNorm(guess, NORM_2, &norm);
PetscPrintf(0, "guess norm = %g\n", norm);
*/
// double errnorm;
// double exsolnorm;
// Inverse solver set up
// std::cout << RED"Setting up Inverse Solver"NRM << std::endl;
parametricWaveInverse* hyperInv = new parametricWaveInverse;
// std::cout << GRN"Finished setting up Inverse Solver"NRM << std::endl;
hyperInv->setTimeStepper(ts); // set the timestepper
hyperInv->setForwardInitialConditions(initialDisplacement, initialVelocity);
// std::cout << RED"Setting initial guess"NRM << std::endl;
// hyperInv->setInitialGuess(truth);// set the initial guess
hyperInv->setInitialGuess(guess);// set the initial guess
// std::cout << GRN"Done setting initial guess"NRM << std::endl;
hyperInv->setRegularizationParameter(beta); // set the regularization paramter
hyperInv->setAdjoints(solvec); // set the data for the problem
// hyperInv->setForceBasis(fBasis);
hyperInv->setNumberOfParameter(numParams);
// std::cout << RED"Initializing Inverse Solver"NRM << std::endl;
hyperInv->init(); // initialize the inverse solver
// if (!rank)
// std::cout << RED"Starting Inverse Solve"NRM << std::endl;
startTime = MPI_Wtime();
hyperInv->solve(); // solve
endTime = MPI_Wtime();
// if (!rank)
// std::cout << GRN"FINISHED HESSIAN SOLVE"NRM << std::endl;
hyperInv->getCurrentControl(guess); // get the solution
hyperInv->destroy();
/*
for (int i=0; i<solvec.size(); i++) {
if (solvec[i] != NULL) {
VecDestroy(solvec[i]);
}
}
solvec.clear();
*/
// VecView(guess, 0);
if (!rank)
std::cout << std::endl << "Error Norms " << std::endl;
Vec Err;
double gtNorm, solNorm, errNorm;
VecDuplicate(guess, &Err);
VecWAXPY(Err, -1.0, guess, alpha);
VecNorm(alpha, NORM_2, >Norm);
VecNorm(guess, NORM_2, &solNorm);
VecNorm(Err, NORM_2, &errNorm);
if (!rank) {
std::cout << "The norms are " << gtNorm << ", " << solNorm << ", " << errNorm << std::endl;
std::cout << "Relative error is " << errNorm/gtNorm << std::endl;
}
// Now we shall do another forward solve ...
getForces(guess, newF, da, ti, numParams);
Force->setFDynamic(newF);
ts->setInitialDisplacement(initialDisplacement);
ts->setInitialVelocity(initialVelocity);
ts->setAdjoint(false);
ts->clearMonitor();
ts->solve();
std::vector<Vec> solvec2 = ts->getSolution();
ts->destroy();
concatenateVecs(solvec2, nr);
// Now can clear memory ...
for (int i=0; i<solvec2.size(); i++) {
if (solvec2[i] != NULL) {
VecDestroy(solvec2[i]);
}
}
solvec2.clear();
// Now can clear memory ...
for (int i=0; i<newF.size(); i++) {
if (newF[i] != NULL) {
VecDestroy(newF[i]);
}
}
newF.clear();
/*
for (unsigned int i=0; i<truth.size(); i++) {
VecNorm(truth[i], NORM_2, >Norm);
VecNorm(solvec[i], NORM_2, &solNorm);
VecAXPY(solvec[i], -1.0, truth[i]);
VecNorm(solvec[i], NORM_2, &errNorm);
PetscPrintf(0, "Ground truth at timestep %d is %g, %g, %g\n", i, gtNorm, solNorm, errNorm);
// PetscPrintf(0, "Relative Error at timestep %d is %g\n", i, errNorm/gtNorm);
}
*/
VecNorm(gt, NORM_2, >Norm);
VecAXPY(nr, -1.0, gt);
VecNorm(nr, NORM_2, &errNorm);
if (!rank)
std::cout << "Total Relative error on state is " << errNorm/gtNorm << std::endl;
if (!rank)
std::cout << "Wall time is " << endTime - startTime << std::endl;
VecDestroy(gt);
VecDestroy(nr);
VecDestroy(Err);
VecDestroy(alpha);
VecDestroy(guess);
VecDestroy(rho);
VecDestroy(nu);
VecDestroy(initialDisplacement);
VecDestroy(initialVelocity);
DADestroy(da);
PetscFinalize();
}
PetscErrorCode ComputeMatrix(DA da,Mat B)
{
PetscErrorCode ierr;
PetscInt i,j,k,mx,my,mz,xm,ym,zm,xs,ys,zs,dof,k1,k2,k3;
PetscScalar *v,*v_neighbor,Hx,Hy,Hz,HxHydHz,HyHzdHx,HxHzdHy;
MatStencil row,col;
PetscFunctionBegin;
ierr = DAGetInfo(da,0,&mx,&my,&mz,0,0,0,&dof,0,0,0);CHKERRQ(ierr);
/* For simplicity, this example only works on mx=my=mz */
if ( mx != my || mx != mz) SETERRQ3(1,"This example only works with mx %d = my %d = mz %d\n",mx,my,mz);
Hx = 1.0 / (PetscReal)(mx-1); Hy = 1.0 / (PetscReal)(my-1); Hz = 1.0 / (PetscReal)(mz-1);
HxHydHz = Hx*Hy/Hz; HxHzdHy = Hx*Hz/Hy; HyHzdHx = Hy*Hz/Hx;
ierr = PetscMalloc((2*dof*dof+1)*sizeof(PetscScalar),&v);CHKERRQ(ierr);
v_neighbor = v + dof*dof;
ierr = PetscMemzero(v,(2*dof*dof+1)*sizeof(PetscScalar));CHKERRQ(ierr);
k3 = 0;
for (k1=0; k1<dof; k1++){
for (k2=0; k2<dof; k2++){
if (k1 == k2){
v[k3] = 2.0*(HxHydHz + HxHzdHy + HyHzdHx);
v_neighbor[k3] = -HxHydHz;
} else {
v[k3] = k1/(dof*dof); ;
v_neighbor[k3] = k2/(dof*dof);
}
k3++;
}
}
ierr = DAGetCorners(da,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr);
for (k=zs; k<zs+zm; k++){
for (j=ys; j<ys+ym; j++){
for(i=xs; i<xs+xm; i++){
row.i = i; row.j = j; row.k = k;
if (i==0 || j==0 || k==0 || i==mx-1 || j==my-1 || k==mz-1){ /* boudary points */
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr);
} else { /* interior points */
/* center */
col.i = i; col.j = j; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v,INSERT_VALUES);CHKERRQ(ierr);
/* x neighbors */
col.i = i-1; col.j = j; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
col.i = i+1; col.j = j; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
/* y neighbors */
col.i = i; col.j = j-1; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
col.i = i; col.j = j+1; col.k = k;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
/* z neighbors */
col.i = i; col.j = j; col.k = k-1;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
col.i = i; col.j = j; col.k = k+1;
ierr = MatSetValuesBlockedStencil(B,1,&row,1,&col,v_neighbor,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
}
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscFree(v);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormFunction - Evaluates nonlinear function, F(x).
Input Parameters:
. snes - the SNES context
. x - input vector
. ctx - optional user-defined context, as set by SNESSetFunction()
Output Parameter:
. f - function vector
Note:
The user-defined context can contain any application-specific
data needed for the function evaluation.
*/
PetscErrorCode FormFunction(SNES snes,Vec x,Vec f,void *ctx)
{
ApplicationCtx *user = (ApplicationCtx*) ctx;
DA da = user->da;
PetscScalar *xx,*ff,*FF,d;
PetscErrorCode ierr;
PetscInt i,M,xs,xm;
Vec xlocal;
PetscFunctionBegin;
ierr = DAGetLocalVector(da,&xlocal);CHKERRQ(ierr);
/*
Scatter ghost points to local vector, using the 2-step process
DAGlobalToLocalBegin(), DAGlobalToLocalEnd().
By placing code between these two statements, computations can
be done while messages are in transition.
*/
ierr = DAGlobalToLocalBegin(da,x,INSERT_VALUES,xlocal);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da,x,INSERT_VALUES,xlocal);CHKERRQ(ierr);
/*
Get pointers to vector data.
- The vector xlocal includes ghost point; the vectors x and f do
NOT include ghost points.
- Using DAVecGetArray() allows accessing the values using global ordering
*/
ierr = DAVecGetArray(da,xlocal,&xx);CHKERRQ(ierr);
ierr = DAVecGetArray(da,f,&ff);CHKERRQ(ierr);
ierr = DAVecGetArray(da,user->F,&FF);CHKERRQ(ierr);
/*
Get local grid boundaries (for 1-dimensional DA):
xs, xm - starting grid index, width of local grid (no ghost points)
*/
ierr = DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DAGetInfo(da,PETSC_NULL,&M,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,
PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
/*
Set function values for boundary points; define local interior grid point range:
xsi - starting interior grid index
xei - ending interior grid index
*/
if (xs == 0) { /* left boundary */
ff[0] = xx[0];
xs++;xm--;
}
if (xs+xm == M) { /* right boundary */
ff[xs+xm-1] = xx[xs+xm-1] - 1.0;
xm--;
}
/*
Compute function over locally owned part of the grid (interior points only)
*/
d = 1.0/(user->h*user->h);
for (i=xs; i<xs+xm; i++) {
ff[i] = d*(xx[i-1] - 2.0*xx[i] + xx[i+1]) + xx[i]*xx[i] - FF[i];
}
/*
Restore vectors
*/
ierr = DAVecRestoreArray(da,xlocal,&xx);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da,f,&ff);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da,user->F,&FF);CHKERRQ(ierr);
ierr = DARestoreLocalVector(da,&xlocal);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
SNES snes; /* nonlinear solver */
Vec Hu,r; /* solution, residual vectors */
Mat J; /* Jacobian matrix */
AppCtx user; /* user-defined work context */
PetscInt its, i, tmpxs, tmpxm; /* iteration count, index, etc. */
PetscReal tmp1, tmp2, tmp3, tmp4, tmp5,
errnorms[2], descaleNode[2];
PetscTruth eps_set = PETSC_FALSE, dump = PETSC_FALSE, exactinitial = PETSC_FALSE,
snes_mf_set, snes_fd_set;
MatFDColoring matfdcoloring = 0;
ISColoring iscoloring;
SNESConvergedReason reason; /* Check convergence */
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &user.rank); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"BODVARDSSON solves for thickness and velocity in 1D, steady ice stream\n"
" [run with -help for info and options]\n");CHKERRQ(ierr);
user.n = 3.0; /* Glen flow law exponent */
user.secpera = 31556926.0;
user.rho = 910.0; /* kg m^-3 */
user.rhow = 1028.0; /* kg m^-3 */
user.g = 9.81; /* m s^-2 */
/* ask Test N for its parameters, but only those we need to solve */
ierr = params_exactN(&(user.H0), &tmp1, &(user.xc), &tmp2, &tmp3, &tmp4, &tmp5,
&(user.Txc)); CHKERRQ(ierr);
/* regularize using strain rate of 1/xc per year */
user.epsilon = (1.0 / user.secpera) / user.xc;
/* tools for non-dimensionalizing to improve equation scaling */
user.scaleNode[0] = 1000.0; user.scaleNode[1] = 100.0 / user.secpera;
ierr = PetscOptionsTruth("-snes_mf","","",PETSC_FALSE,&snes_mf_set,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-snes_fd","","",PETSC_FALSE,&snes_fd_set,NULL);CHKERRQ(ierr);
if (!snes_mf_set && !snes_fd_set) {
PetscPrintf(PETSC_COMM_WORLD,
"\n***ERROR: bodvardsson needs one or zero of '-snes_mf', '-snes_fd'***\n\n"
"USAGE FOLLOWS ...\n\n%s",help);
PetscEnd();
}
if (snes_fd_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" using approximate Jacobian; finite-differencing using coloring\n");
CHKERRQ(ierr);
} else if (snes_mf_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" matrix free; no preconditioner\n"); CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" true Jacobian\n"); CHKERRQ(ierr);
}
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,
"bodvardsson program options",__FILE__);CHKERRQ(ierr);
{
ierr = PetscOptionsTruth("-bod_up_one","","",PETSC_FALSE,&user.upwind1,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-bod_exact_init","","",PETSC_FALSE,&exactinitial,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-bod_dump",
"dump out exact and approximate solution and residual, as asci","",
dump,&dump,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-bod_epsilon","regularization (a strain rate in units of 1/a)","",
user.epsilon * user.secpera,&user.epsilon,&eps_set);CHKERRQ(ierr);
if (eps_set) user.epsilon *= 1.0 / user.secpera;
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
/* Create machinery for parallel grid management (DA), nonlinear solver (SNES),
and Vecs for fields (solution, RHS). Note default Mx=46 grid points means
dx=10 km. Also degrees of freedom = 2 (thickness and velocity
at each point) and stencil radius = ghost width = 2 for 2nd-order upwinding. */
user.solnghostwidth = 2;
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,-46,2,user.solnghostwidth,PETSC_NULL,&user.da);
CHKERRQ(ierr);
ierr = DASetUniformCoordinates(user.da,0.0,user.xc,
PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DASetFieldName(user.da,0,"ice thickness [non-dimensional]"); CHKERRQ(ierr);
ierr = DASetFieldName(user.da,1,"ice velocity [non-dimensional]"); CHKERRQ(ierr);
ierr = DAGetInfo(user.da,PETSC_IGNORE,&user.Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
ierr = DAGetCorners(user.da,&user.xs,PETSC_NULL,PETSC_NULL,&user.xm,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
user.dx = user.xc / (PetscReal)(user.Mx-1);
/* another DA for scalar parameters, with same length */
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,user.Mx,1,1,PETSC_NULL,&user.scalarda);CHKERRQ(ierr);
ierr = DASetUniformCoordinates(user.scalarda,0.0,user.xc,
PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
/* check that parallel layout of scalar DA is same as dof=2 DA */
ierr = DAGetCorners(user.scalarda,&tmpxs,PETSC_NULL,PETSC_NULL,&tmpxm,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
if ((tmpxs != user.xs) || (tmpxm != user.xm)) {
PetscPrintf(PETSC_COMM_SELF,
"\n***ERROR: rank %d gets different ownership range for the two DAs! ENDING ...***\n\n",
user.rank);
PetscEnd();
}
ierr = PetscPrintf(PETSC_COMM_WORLD,
" Mx = %D points, dx = %.3f m\n H0 = %.2f m, xc = %.2f km, Txc = %.5e Pa m\n",
user.Mx, user.dx, user.H0, user.xc/1000.0, user.Txc);CHKERRQ(ierr);
/* Extract/allocate global vectors from DAs and duplicate for remaining same types */
ierr = DACreateGlobalVector(user.da,&Hu);CHKERRQ(ierr);
ierr = VecSetBlockSize(Hu,2);CHKERRQ(ierr);
ierr = VecDuplicate(Hu,&r);CHKERRQ(ierr); /* inherits block size */
ierr = VecDuplicate(Hu,&user.Huexact);CHKERRQ(ierr); /* ditto */
ierr = DACreateGlobalVector(user.scalarda,&user.M);CHKERRQ(ierr);
ierr = VecDuplicate(user.M,&user.Bstag);CHKERRQ(ierr);
ierr = VecDuplicate(user.M,&user.beta);CHKERRQ(ierr);
ierr = DASetLocalFunction(user.da,(DALocalFunction1)scshell);CHKERRQ(ierr);
ierr = DASetLocalJacobian(user.da,(DALocalFunction1)BodJacobianMatrixLocal);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
ierr = SNESSetFunction(snes,r,SNESDAFormFunction,&user);CHKERRQ(ierr);
/* setting up a matrix is only actually needed for -snes_fd case */
ierr = DAGetMatrix(user.da,MATAIJ,&J);CHKERRQ(ierr);
if (snes_fd_set) {
/* tools needed so DA can use sparse matrix for its F.D. Jacobian approx */
ierr = DAGetColoring(user.da,IS_COLORING_GLOBAL,MATAIJ,&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringCreate(J,iscoloring,&matfdcoloring);CHKERRQ(ierr);
ierr = ISColoringDestroy(iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringSetFunction(matfdcoloring,
(PetscErrorCode (*)(void))SNESDAFormFunction,&user);CHKERRQ(ierr);
ierr = MatFDColoringSetFromOptions(matfdcoloring);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes,J,J,SNESDefaultComputeJacobianColor,matfdcoloring);CHKERRQ(ierr);
} else {
ierr = SNESSetJacobian(snes,J,J,SNESDAComputeJacobian,&user);CHKERRQ(ierr);
}
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
/* the the Bodvardsson (1955) exact solution allows setting M(x), B(x), beta(x), T(xc) */
ierr = FillDistributedParams(&user);CHKERRQ(ierr);
/* the exact thickness and exact ice velocity (user.uHexact) are known from Bodvardsson (1955) */
ierr = FillExactSoln(&user); CHKERRQ(ierr);
if (exactinitial) {
ierr = PetscPrintf(PETSC_COMM_WORLD," using exact solution as initial guess\n");
CHKERRQ(ierr);
/* the initial guess is the exact continuum solution */
ierr = VecCopy(user.Huexact,Hu); CHKERRQ(ierr);
} else {
ierr = FillInitial(&user, &Hu); CHKERRQ(ierr);
}
/************ SOLVE NONLINEAR SYSTEM ************/
/* recall that RHS r is used internally by KSP, and is set by the SNES */
for (i = 0; i < 2; i++) descaleNode[i] = 1.0 / user.scaleNode[i];
ierr = VecStrideScaleAll(Hu,descaleNode); CHKERRQ(ierr); /* de-dimensionalize initial guess */
ierr = SNESSolve(snes,PETSC_NULL,Hu);CHKERRQ(ierr);
ierr = VecStrideScaleAll(Hu,user.scaleNode); CHKERRQ(ierr); /* put back in "real" scale */
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = SNESGetConvergedReason(snes,&reason);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" %s Number of Newton iterations = %D\n",
SNESConvergedReasons[reason],its);CHKERRQ(ierr);
if (dump) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing combined result Hu\n");CHKERRQ(ierr);
ierr = VecView(Hu,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing combined exact result Huexact\n");CHKERRQ(ierr);
ierr = VecView(user.Huexact,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing final combined residual at Hu\n");CHKERRQ(ierr);
ierr = VecView(r,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
}
/* evaluate error relative to exact solution */
ierr = VecAXPY(Hu,-1.0,user.Huexact); CHKERRQ(ierr); /* Hu = - Huexact + Hu */
ierr = VecStrideNormAll(Hu,NORM_INFINITY,errnorms); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"(dx,errHinf,erruinf) %.3f %.4e %.4e\n",
user.dx,errnorms[0],errnorms[1]*user.secpera);CHKERRQ(ierr);
ierr = VecDestroy(Hu);CHKERRQ(ierr);
ierr = VecDestroy(r);CHKERRQ(ierr);
ierr = VecDestroy(user.Huexact);CHKERRQ(ierr);
ierr = VecDestroy(user.M);CHKERRQ(ierr);
ierr = VecDestroy(user.Bstag);CHKERRQ(ierr);
ierr = VecDestroy(user.beta);CHKERRQ(ierr);
ierr = MatDestroy(J); CHKERRQ(ierr);
ierr = SNESDestroy(snes);CHKERRQ(ierr);
ierr = DADestroy(user.da);CHKERRQ(ierr);
ierr = DADestroy(user.scalarda);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc, char **argv)
{
PetscInitialize(&argc, &argv, "elas.opt", help);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
int Ns = 32;
unsigned int dof = 3;
char problemName[PETSC_MAX_PATH_LEN];
char filename[PETSC_MAX_PATH_LEN];
double t0 = 0.0;
double dt = 0.1;
double t1 = 1.0;
double beta = 0.000001;
double gamma = 0.0; // percent of noise added ...
// double dtratio = 1.0;
DA da; // Underlying scalar DA - for scalar properties
DA da3d; // Underlying vector DA - for vector properties
Vec rho; // density - elemental scalar
Vec lambda; // Lame parameter - lambda - elemental scalar
Vec mu; // Lame parameter - mu - elemental scalar
Vec fibers; // Fiber orientations - nodal vector (3-dof)
Vec fibersElemental; // for IO. will be destroyed. - elemental vector (3-dof)
std::vector<Vec> tau; // the scalar activation - nodal scalar
// Initial conditions
Vec initialDisplacement;
Vec initialVelocity;
timeInfo ti;
// get Ns
CHKERRQ ( PetscOptionsGetInt(0,"-Ns",&Ns,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-t0",&t0,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-t1",&t1,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-dt",&dt,0) );
CHKERRQ ( PetscOptionsGetScalar(0,"-beta",&beta,0) );
CHKERRQ ( PetscOptionsGetString(PETSC_NULL,"-pn",problemName,PETSC_MAX_PATH_LEN-1,PETSC_NULL));
if (!rank) {
std::cout << "Problem size is " << Ns+1 << " spatially and NT = " << (int)ceil(1.0/dt) << std::endl;
}
// Time info for timestepping
ti.start = t0;
ti.stop = t1;
ti.step = dt;
// create DA
CHKERRQ ( DACreate3d ( PETSC_COMM_WORLD, DA_NONPERIODIC, DA_STENCIL_BOX,
Ns+1, Ns+1, Ns+1, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE,
1, 1, 0, 0, 0, &da) );
CHKERRQ ( DACreate3d ( PETSC_COMM_WORLD, DA_NONPERIODIC, DA_STENCIL_BOX,
Ns+1, Ns+1, Ns+1, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE,
dof, 1, 0, 0, 0, &da3d) );
elasMass *Mass = new elasMass(feMat::PETSC); // Mass Matrix
elasStiffness *Stiffness = new elasStiffness(feMat::PETSC); // Stiffness matrix
raleighDamping *Damping = new raleighDamping(feMat::PETSC); // Damping Matrix
cardiacDynamic *Force = new cardiacDynamic(feVec::PETSC); // Force Vector
// create vectors
CHKERRQ( DACreateGlobalVector(da, &rho) );
CHKERRQ( DACreateGlobalVector(da, &mu) );
CHKERRQ( DACreateGlobalVector(da, &lambda) );
CHKERRQ( DACreateGlobalVector(da3d, &initialDisplacement) );
CHKERRQ( DACreateGlobalVector(da3d, &initialVelocity) );
// Set initial conditions
CHKERRQ( VecSet ( initialDisplacement, 0.0) );
CHKERRQ( VecSet ( initialVelocity, 0.0) );
VecZeroEntries( mu );
VecZeroEntries( lambda );
VecZeroEntries( rho );
int x, y, z, m, n, p;
int mx,my,mz, xne, yne, zne;
CHKERRQ( DAGetCorners(da, &x, &y, &z, &m, &n, &p) );
CHKERRQ( DAGetInfo(da,0, &mx, &my, &mz, 0,0,0,0,0,0,0) );
if (x+m == mx) {
xne=m-1;
} else {
xne=m;
}
if (y+n == my) {
yne=n-1;
} else {
yne=n;
}
if (z+p == mz) {
zne=p-1;
} else {
zne=p;
}
double acx,acy,acz;
double hx = 1.0/((double)Ns);
// SET MATERIAL PROPERTIES ...
// @todo - Write routines to read/write in Parallel
// allocate for temporary buffers ...
unsigned int elemSize = Ns*Ns*Ns;
// std::cout << "Elem size is " << elemSize << std::endl;
unsigned int nodeSize = (Ns+1)*(Ns+1)*(Ns+1);
unsigned char *tmp_mat = new unsigned char[elemSize];
double *tmp_tau = new double[dof*elemSize];
// generate filenames & read in the raw arrays first ...
std::ifstream fin;
sprintf(filename, "%s.%d.img", problemName, Ns);
fin.open(filename, std::ios::binary); fin.read((char *)tmp_mat, elemSize); fin.close();
// Set Elemental material properties
PetscScalar ***muArray, ***lambdaArray, ***rhoArray;
CHKERRQ(DAVecGetArray(da, mu, &muArray));
CHKERRQ(DAVecGetArray(da, lambda, &lambdaArray));
CHKERRQ(DAVecGetArray(da, rho, &rhoArray));
// assign material properties ...
// myo, tissue
// nu = 0.49, 0.45
// E = 10000, 1000
// rho = 1.0, 0.1
// std::cout << "Setting Elemental properties." << std::endl;
// loop through all elements ...
for (int k=z; k<z+zne; k++) {
for (int j=y; j<y+yne; j++) {
for (int i=x; i<x+xne; i++) {
int indx = k*Ns*Ns + j*Ns + i;
if ( tmp_mat[indx] ) {
muArray[k][j][i] = 344.82; //3355.7;
lambdaArray[k][j][i] = 3103.448;// 164429.53;
rhoArray[k][j][i] = 1.0;
} else {
muArray[k][j][i] = 344.82;
lambdaArray[k][j][i] = 3103.448;
rhoArray[k][j][i] = 1.0;
}
} // end i
} // end j
} // end k
// std::cout << "Finished Elemental loop." << std::endl;
CHKERRQ( DAVecRestoreArray ( da, mu, &muArray ) );
CHKERRQ( DAVecRestoreArray ( da, lambda, &lambdaArray ) );
CHKERRQ( DAVecRestoreArray ( da, rho, &rhoArray ) );
// std::cout << "Finished restoring arrays" << std::endl;
// delete temporary buffers
delete [] tmp_mat;
// Now set the activation ...
unsigned int numSteps = (unsigned int)(ceil(( ti.stop - ti.start)/ti.step));
// tau = (Vec *) new char*[numSteps+1];
// std::cout << "Numsteps is " << numSteps << std::endl;
Vec tauVec, tmpTau;
CHKERRQ( DACreateGlobalVector(da3d, &tmpTau) );
#ifdef __DEBUG__
if (!rank) {
std::cout << x << ", " << y << ", " << z << " + " << xne << ", " << yne << ", " << zne << std::endl;
}
#endif
PetscScalar ***tauArray;
double tauNorm;
for (unsigned int t=0; t<numSteps+1; t++) {
CHKERRQ( DACreateGlobalVector(da3d, &tauVec) );
CHKERRQ( VecSet( tmpTau, 0.0));
CHKERRQ(DAVecGetArray(da3d, tmpTau, &tauArray));
// std::cout << "Setting force vectors" << std::endl;
sprintf(filename, "%s.%d.%.3d.fld", problemName, Ns, t);
// std::cout << "Reading force file " << filename << std::endl;
fin.open(filename); fin.read((char *)tmp_tau, dof*elemSize*sizeof(double)); fin.close();
for (int k = z; k < z + zne ; k++) {
for (int j = y; j < y + yne; j++) {
for (int i = x; i < x + xne; i++) {
int indx = dof*(k*Ns*Ns + j*Ns + i);
tauArray[k][j][dof*i] = tmp_tau[indx];
tauArray[k][j][dof*i+1] = tmp_tau[indx+1];
tauArray[k][j][dof*i+2] = tmp_tau[indx+2];
}
}
}
// std::cout << CYN"\tFinished elemental loop"NRM << std::endl;
CHKERRQ( DAVecRestoreArray ( da3d, tmpTau, &tauArray ) );
// std::cout << "Converting to Nodal Vector" << std::endl;
// VecNorm(tmpTau, NORM_2, &tauNorm);
// tauNorm = tauNorm/pow(Ns,1.5);
// std::cout << "Elemental Norm is " << tauNorm << std::endl;
// std::cout << rank << " Converting to Nodal" << std::endl;
elementToNode(da3d, tmpTau, tauVec);
/*
VecNorm(tauVec, NORM_2, &tauNorm);
tauNorm = tauNorm/pow(Ns,1.5);
std::cout << "Nodal Norm is " << tauNorm << std::endl;
*/
// std::cout << rank << " Done converting to Nodal Vector" << std::endl;
tau.push_back(tauVec);
}
//if (!rank) {
// std::cout << "Finished setting forces" << std::endl;
// }
// CHKERRQ( VecDestroy( tmpTau ) );
delete [] tmp_tau;
// DONE - SET MATERIAL PROPERTIES ...
// Setup Matrices and Force Vector ...
Mass->setProblemDimensions(1.0, 1.0, 1.0);
Mass->setDA(da3d);
Mass->setDof(dof);
Mass->setDensity(rho);
Stiffness->setProblemDimensions(1.0, 1.0, 1.0);
Stiffness->setDA(da3d);
Stiffness->setDof(dof);
Stiffness->setLame(lambda, mu);
Damping->setAlpha(0.0);
Damping->setBeta(0.00075);
Damping->setMassMatrix(Mass);
Damping->setStiffnessMatrix(Stiffness);
Damping->setDA(da3d);
Damping->setDof(dof);
// Force Vector
Force->setProblemDimensions(1.0,1.0,1.0);
Force->setDA(da3d);
// Force->setActivationVec(tau);
// Force->setFiberOrientations(fibers);
Force->setFDynamic(tau);
Force->setTimeInfo(&ti);
// Newmark time stepper ...
newmark *ts = new newmark;
ts->setMassMatrix(Mass);
ts->setDampingMatrix(Damping);
ts->setStiffnessMatrix(Stiffness);
ts->damp(false);
ts->setTimeFrames(1);
ts->setForceVector(Force);
ts->setInitialDisplacement(initialDisplacement);
ts->setInitialVelocity(initialVelocity);
ts->storeVec(true);
ts->setTimeInfo(&ti);
ts->setAdjoint(false); // set if adjoint or forward
ts->init(); // initialize IMPORTANT
if (!rank)
std::cout << RED"Starting Newmark Solve"NRM << std::endl;
ts->solve();// solve
if (!rank)
std::cout << GRN"Done Newmark"NRM << std::endl;
std::vector<Vec> solvec = ts->getSolution();
/* Set very initial guess for the inverse problem*/
/*
PetscRandom rctx;
PetscRandomCreate(PETSC_COMM_WORLD,&rctx);
PetscRandomSetFromOptions(rctx);
VecSetRandom(guess,rctx);
VecNorm(guess,NORM_INFINITY,&norm);
PetscPrintf(0,"guess norm = %g\n",norm);
*/
double errnorm;
double exsolnorm;
Vec guess;
Vec truth;
Vec Err;
concatenateVecs(solvec, guess);
concatenateVecs(tau, truth);
iC(VecNorm(truth, NORM_2, &exsolnorm));
/*
std::cout << "Forward solver solution size is " << solvec.size() << std::endl;
std::cout << "Forward solver solution norm is " << exsolnorm << std::endl;
*/
VecZeroEntries(guess);
// Inverse solver set up
hyperbolicInverse *hyperInv = new hyperbolicInverse;
hyperInv->setForwardInitialConditions(initialDisplacement, initialVelocity);
hyperInv->setTimeStepper(ts); // set the timestepper
hyperInv->setInitialGuess(guess);// set the initial guess
// hyperInv->setInitialGuess(truth);// set the initial guess
hyperInv->setRegularizationParameter(beta); // set the regularization paramter
hyperInv->setObservations(solvec); // set the data for the problem
hyperInv->init(); // initialize the inverse solver
hyperInv->solve(); // solve
hyperInv->getCurrentControl(guess); // get the solution
// see the error in the solution relative to the actual solution
VecDuplicate(truth, &Err);
iC(VecZeroEntries(Err));
iC(VecWAXPY(Err, -1.0, guess, truth));
iC(VecNorm(Err, NORM_2, &errnorm));
PetscPrintf(0,"errr in inverse = %g\n", errnorm/exsolnorm);
PetscFinalize();
}
/* Compute production variables, update Observations vector */
extern PetscErrorCode DefiantBlackOil3PhProduction(BlackOilReservoirSimulation* MySim)
{
PetscErrorCode ierr;
PetscInt mx, my, mz, xm, ym, zm, xs, ys, zs;
/* Well handling variables */
PetscInt PerfIDMine, WellID;
PetscFunctionBegin;
/* Get dimensions and extents of the local vectors */
ierr = DAGetInfo(MySim->SimDA, 0, &mx, &my, &mz, 0, 0, 0, 0, 0, 0, 0);CHKERRQ(ierr);
ierr = DAGetCorners(MySim->SimDA, &xs, &ys, &zs, &xm, &ym, &zm);CHKERRQ(ierr);
/* Sync everything before we start */
ierr = DefiantBlackOilComputePerfIndicesSyncPerfs(MySim);CHKERRQ(ierr);
for (WellID = 0; WellID < MySim->NumberOfWells; WellID++) {
for (PerfIDMine = 0; PerfIDMine
< (MySim->Wells[WellID]).NumberOfPerforations; PerfIDMine++) {
if (MySim->Wells[WellID].Perforations[PerfIDMine].IsActive == PETSC_TRUE ){
if (MySim->Wells[WellID].Perforations[PerfIDMine].Constraint == FLOW_RATE_CONSTRAINT) {
MySim->Wells[WellID].Perforations[PerfIDMine].BHPo =
MySim->Wells[WellID].Perforations[PerfIDMine].Bo
/ (MySim->Wells[WellID].Perforations[PerfIDMine].h1
* MySim->Wells[WellID].Perforations[PerfIDMine].h2
* MySim->Wells[WellID].Perforations[PerfIDMine].h3)
* MySim->Wells[WellID].Perforations[PerfIDMine].Muo
/ MySim->Wells[WellID].Perforations[PerfIDMine].Kro
/ MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex
+ MySim->Wells[WellID].Perforations[PerfIDMine].Po
+ MySim->GravAcc * MySim->Wells[WellID].Perforations[PerfIDMine].Rhoo
* (MySim->Wells[WellID].Perforations[PerfIDMine].zbh
- MySim->Wells[WellID].Perforations[PerfIDMine].x3);
MySim->Wells[WellID].Perforations[PerfIDMine].BHPw =
MySim->Wells[WellID].Perforations[PerfIDMine].Bw
/ (MySim->Wells[WellID].Perforations[PerfIDMine].h1
* MySim->Wells[WellID].Perforations[PerfIDMine].h2
* MySim->Wells[WellID].Perforations[PerfIDMine].h3)
* MySim->Wells[WellID].Perforations[PerfIDMine].Muw
/ MySim->Wells[WellID].Perforations[PerfIDMine].Krw
/ MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex
+ MySim->Wells[WellID].Perforations[PerfIDMine].Pw
+ MySim->GravAcc * MySim->Wells[WellID].Perforations[PerfIDMine].Rhow
* (MySim->Wells[WellID].Perforations[PerfIDMine].zbh
- MySim->Wells[WellID].Perforations[PerfIDMine].x3);
MySim->Wells[WellID].Perforations[PerfIDMine].BHPg =
MySim->Wells[WellID].Perforations[PerfIDMine].Bg
/ (MySim->Wells[WellID].Perforations[PerfIDMine].h1
* MySim->Wells[WellID].Perforations[PerfIDMine].h2
* MySim->Wells[WellID].Perforations[PerfIDMine].h3)
* MySim->Wells[WellID].Perforations[PerfIDMine].Mug
/ MySim->Wells[WellID].Perforations[PerfIDMine].Krg
/ MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex
+ MySim->Wells[WellID].Perforations[PerfIDMine].Pg
+ MySim->GravAcc * MySim->Wells[WellID].Perforations[PerfIDMine].Rhog
* (MySim->Wells[WellID].Perforations[PerfIDMine].zbh
- MySim->Wells[WellID].Perforations[PerfIDMine].x3);
} else if (MySim->Wells[WellID].Perforations[PerfIDMine].Constraint == BHP_CONSTRAINT) {
MySim->Wells[WellID].Perforations[PerfIDMine].Qo =
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex
* MySim->Wells[WellID].Perforations[PerfIDMine].Kro
/ MySim->Wells[WellID].Perforations[PerfIDMine].Muo
* (MySim->Wells[WellID].Perforations[PerfIDMine].BHPo
- MySim->Wells[WellID].Perforations[PerfIDMine].Po
- MySim->GravAcc * MySim->Wells[WellID].Perforations[PerfIDMine].Rhoo
* (MySim->Wells[WellID].Perforations[PerfIDMine].zbh
- MySim->Wells[WellID].Perforations[PerfIDMine].x3))
* MySim->Wells[WellID].Perforations[PerfIDMine].h1
* MySim->Wells[WellID].Perforations[PerfIDMine].h2
* MySim->Wells[WellID].Perforations[PerfIDMine].h3
/ MySim->Wells[WellID].Perforations[PerfIDMine].Bo;
MySim->Wells[WellID].Perforations[PerfIDMine].Qw =
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex
* MySim->Wells[WellID].Perforations[PerfIDMine].Krw
/ MySim->Wells[WellID].Perforations[PerfIDMine].Muw
* (MySim->Wells[WellID].Perforations[PerfIDMine].BHPw
- MySim->Wells[WellID].Perforations[PerfIDMine].Pw
- MySim->GravAcc * MySim->Wells[WellID].Perforations[PerfIDMine].Rhow
* (MySim->Wells[WellID].Perforations[PerfIDMine].zbh
- MySim->Wells[WellID].Perforations[PerfIDMine].x3))
* MySim->Wells[WellID].Perforations[PerfIDMine].h1
* MySim->Wells[WellID].Perforations[PerfIDMine].h2
* MySim->Wells[WellID].Perforations[PerfIDMine].h3
/ MySim->Wells[WellID].Perforations[PerfIDMine].Bw;
MySim->Wells[WellID].Perforations[PerfIDMine].Qg =
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex
* MySim->Wells[WellID].Perforations[PerfIDMine].Krg
/ MySim->Wells[WellID].Perforations[PerfIDMine].Mug
* (MySim->Wells[WellID].Perforations[PerfIDMine].BHPg
- MySim->Wells[WellID].Perforations[PerfIDMine].Pg
- MySim->GravAcc * MySim->Wells[WellID].Perforations[PerfIDMine].Rhog
* (MySim->Wells[WellID].Perforations[PerfIDMine].zbh
- MySim->Wells[WellID].Perforations[PerfIDMine].x3))
* MySim->Wells[WellID].Perforations[PerfIDMine].h1
* MySim->Wells[WellID].Perforations[PerfIDMine].h2
* MySim->Wells[WellID].Perforations[PerfIDMine].h3
/ MySim->Wells[WellID].Perforations[PerfIDMine].Bg;
}
}
}
}
PetscFunctionReturn(0);
}
/*
FormJacobian - Evaluates Jacobian matrix.
Input Parameters:
. snes - the SNES context
. x - input vector
. dummy - optional user-defined context (not used here)
Output Parameters:
. jac - Jacobian matrix
. B - optionally different preconditioning matrix
. flag - flag indicating matrix structure
*/
PetscErrorCode FormJacobian(SNES snes,Vec x,Mat *jac,Mat *B,MatStructure*flag,void *ctx)
{
ApplicationCtx *user = (ApplicationCtx*) ctx;
PetscScalar *xx,d,A[3];
PetscErrorCode ierr;
PetscInt i,j[3],M,xs,xm;
DA da = user->da;
PetscFunctionBegin;
/*
Get pointer to vector data
*/
ierr = DAVecGetArray(da,x,&xx);CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
/*
Get range of locally owned matrix
*/
ierr = DAGetInfo(da,PETSC_NULL,&M,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,
PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
/*
Determine starting and ending local indices for interior grid points.
Set Jacobian entries for boundary points.
*/
if (xs == 0) { /* left boundary */
i = 0; A[0] = 1.0;
ierr = MatSetValues(*jac,1,&i,1,&i,A,INSERT_VALUES);CHKERRQ(ierr);
xs++;xm--;
}
if (xs+xm == M) { /* right boundary */
i = M-1; A[0] = 1.0;
ierr = MatSetValues(*jac,1,&i,1,&i,A,INSERT_VALUES);CHKERRQ(ierr);
xm--;
}
/*
Interior grid points
- Note that in this case we set all elements for a particular
row at once.
*/
d = 1.0/(user->h*user->h);
for (i=xs; i<xs+xm; i++) {
j[0] = i - 1; j[1] = i; j[2] = i + 1;
A[0] = A[2] = d; A[1] = -2.0*d + 2.0*xx[i];
ierr = MatSetValues(*jac,1,&i,3,j,A,INSERT_VALUES);CHKERRQ(ierr);
}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
Also, restore vector.
*/
ierr = MatAssemblyBegin(*jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da,x,&xx);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
*flag = SAME_NONZERO_PATTERN;
PetscFunctionReturn(0);
}
PetscErrorCode vizGA2DA()
{
PetscErrorCode ierr;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
int d1 = 40, d2 = 50;
DA da;
Vec vec;
const PetscInt *lx, *ly, *lz;
PetscInt m,n,p;
DALocalInfo info;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr);
ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr);
/**/
ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
for (int i = 0; i < m; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]);
}
for (int i = 0; i < n; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]);
}
/**/
int ga = GA_Create_handle();
int ndim = 2;
int dims[2] = {d2,d1};
GA_Set_data(ga,2,dims,MT_DBL);
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < n; i++ )
{
map[i] = ly[i-1] + map[i-1];
}
map[n] = 0;
for( int i = n+1; i < m+n; i++ )
{
map[i] = lx[i-n-1] + map[i-1];
}
/* correct ordering, but nodeid's dont line up with mpi rank for petsc's da
* DA: +---+---+ GA: +---+---+
* +-2-+-3-+ +-1-+-3-+
* +---+---+ +---+---+
* +-0-+-1-+ +-0-+-2-+
* +---+---+ +---+---+
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < m; i++ )
{
map[i] = lx[i] + map[i-1];
}
map[m] = 0;
for( int i = m+1; i < m+n; i++ )
{
map[i] = ly[i-m] + map[i-1];
}
*/
int block[2] = {n,m};
GA_Set_irreg_distr(ga,map,block);
ierr = GA_Allocate( ga );
if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr);
if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr);
if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1);
GA_Print_distribution(ga);
int lo[2], hi[2];
NGA_Distribution(ga,rank,lo,hi);
if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 ||
lo[0] != info.ys || hi[0] != info.ys+info.ym-1 )
{
PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n",
rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1);
}
PetscBarrier(0);
PetscSynchronizedFlush(PETSC_COMM_WORLD);
AO ao;
DAGetAO(da,&ao);
if( rank == 0 )
{
int *idx, len = d1*d2;
PetscReal *val;
PetscMalloc(sizeof(PetscReal)*len, &val);
PetscMalloc(sizeof(int)*len, &idx);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
idx[i + d1*j] = i + d1*j;
val[i + d1*j] = i + d1*j;
}
}
AOApplicationToPetsc(ao,len,idx);
VecSetValues(vec,len,idx,val,INSERT_VALUES);
int a[2], b[2],ld[1]={0};
double c = 0;
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
a[0] = j;
a[1] = i;
// printf("%5.0f ",c);
NGA_Put(ga,a,a,&c,ld);
c++;
}
}
}
// GA_Print(ga);
VecAssemblyBegin(vec);
VecAssemblyEnd(vec);
int ld;
double *ptr;
NGA_Access(ga,lo,hi,&ptr,&ld);
PetscReal **d;
int c=0;
ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr);
for (int j = info.ys; j < info.ys+info.ym; ++j)
{
for (int i = info.xs; i < info.xs+info.xm; ++i)
{
if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] )
GA_Error("DA array is not equal to GA array",1);
// printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]);
c++;
}
}
ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr);
c=0;
PetscReal *v;
int start, end;
VecGetOwnershipRange(vec, &start, &end);
VecGetArray( vec, &v );
for( int i = start; i < end; i++)
{
// printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") );
}
VecRestoreArray( vec, &v );
NGA_Release_update(ga,lo,hi);
Vec gada;
VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada);
VecView(gada,PETSC_VIEWER_STDOUT_SELF);
GA_Destroy(ga);
ierr = VecDestroy(vec); CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
extern PetscErrorCode DefiantBlackOilSyncPerfOwners(BlackOilReservoirSimulation* MySim)
{
PetscErrorCode ierr;
PetscInt i, rank, TotalNumberOfPerfs;
PetscInt mx, my, mz, xm, ym, zm, xs, ys, zs;
/* Well handling variables */
PetscInt PerfIDMine, WellID;
PetscInt MyI, MyJ, MyK;
/* temporary array */
PetscInt *InBuffer, *OutBuffer;
PetscFunctionBegin;
/* Get dimensions and extents of the local vectors */
ierr = DAGetInfo(MySim->SimDA, 0, &mx, &my, &mz, 0, 0, 0, 0, 0, 0, 0);CHKERRQ(ierr);
ierr = DAGetCorners(MySim->SimDA, &xs, &ys, &zs, &xm, &ym, &zm);CHKERRQ(ierr);
/* Get the current rank */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);CHKMEMQ;
/* Allocate memory to the in and out Buffers */
/* Find the total number of perforations */
TotalNumberOfPerfs = 0;
for (WellID = 0; WellID < MySim->NumberOfWells; WellID++)
for (PerfIDMine = 0; PerfIDMine < (MySim->Wells[WellID]).NumberOfPerforations; PerfIDMine++)
TotalNumberOfPerfs++;
/* Allocate the In and out Buffers */
ierr = PetscMalloc(sizeof(PetscInt)*TotalNumberOfPerfs, &InBuffer);CHKERRQ(ierr);CHKMEMQ;
ierr = PetscMalloc(sizeof(PetscInt)*TotalNumberOfPerfs, &OutBuffer);CHKERRQ(ierr);CHKMEMQ;
/* Pack all the perf owner location int the in buffer */
i = 0;
for (WellID = 0; WellID < MySim->NumberOfWells; WellID++) {
for (PerfIDMine = 0; PerfIDMine
< (MySim->Wells[WellID]).NumberOfPerforations; PerfIDMine++) {
MyI = (MySim->Wells[WellID]).Perforations[PerfIDMine].I;CHKMEMQ;
MyJ = (MySim->Wells[WellID]).Perforations[PerfIDMine].J;CHKMEMQ;
MyK = (MySim->Wells[WellID]).Perforations[PerfIDMine].K;CHKMEMQ;
if (MyI >= xs && MyI < xs+xm && MyJ >= ys && MyJ < ys+ym && MyK >= zs && MyK < zs+zm)
MySim->Wells[WellID].Perforations[PerfIDMine].OwnerRank = rank;
else
MySim->Wells[WellID].Perforations[PerfIDMine].OwnerRank = -1;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].OwnerRank;
i++;
}
}
/* Make the syncrhonization call */
ierr = MPI_Allreduce(InBuffer, OutBuffer, TotalNumberOfPerfs, MPI_INT, MPI_MAX, PETSC_COMM_WORLD);CHKERRQ(ierr);CHKMEMQ;
/* Now unpack all the perf owner location int the in buffer */
i = 0;
for (WellID = 0; WellID < MySim->NumberOfWells; WellID++) {
for (PerfIDMine = 0; PerfIDMine
< (MySim->Wells[WellID]).NumberOfPerforations; PerfIDMine++) {
MySim->Wells[WellID].Perforations[PerfIDMine].OwnerRank = OutBuffer[i];
i++;
}
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
/*-----------------------------------------------------------------------*/
{
DMMG *dmmg; /* multilevel grid structure */
AppCtx *user; /* user-defined work context */
Parameter *param;
GridInfo grid;
int ierr,result;
MPI_Comm comm;
DA da;
PetscInitialize(&argc,&argv,(char *)0,help);
comm = PETSC_COMM_WORLD;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set up the problem parameters.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscMalloc(sizeof(AppCtx),&user);
CHKERRQ(ierr);
ierr = PetscBagCreate(comm,sizeof(Parameter),&(user->bag));
CHKERRQ(ierr);
user->grid = &grid;
ierr = SetParams(user);
CHKERRQ(ierr);
ierr = ReportParams(user);
CHKERRQ(ierr);
ierr = PetscBagGetData(user->bag,(void**)¶m);
CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create distributed array multigrid object (DMMG) to manage parallel grid and vectors
for principal unknowns (x) and governing residuals (f)
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DMMGCreate(comm,grid.mglevels,user,&dmmg);
CHKERRQ(ierr);
ierr = DACreate2d(comm,grid.periodic,grid.stencil,grid.ni,grid.nj,PETSC_DECIDE,PETSC_DECIDE,grid.dof,grid.stencil_width,0,0,&da);
CHKERRQ(ierr);
ierr = DMMGSetDM(dmmg,(DM)da);
CHKERRQ(ierr);
ierr = DADestroy(da);
CHKERRQ(ierr);
ierr = DAGetInfo(da,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,&(param->pi),&(param->pj),PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
REG_INTG(user->bag,¶m->pi,param->pi ,"procs_x","<DO NOT SET> Processors in the x-direction");
REG_INTG(user->bag,¶m->pj,param->pj ,"procs_y","<DO NOT SET> Processors in the y-direction");
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create user context, set problem data, create vector data structures.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DAGetGlobalVector(da, &(user->Xold));
CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Initialize and solve the nonlinear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = Initialize(dmmg);
CHKERRQ(ierr);
ierr = DoSolve(dmmg);
CHKERRQ(ierr);
if (param->verify) result = param->verify_result;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free work space.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DARestoreGlobalVector(da, &(user->Xold));
CHKERRQ(ierr);
ierr = PetscBagDestroy(user->bag);
CHKERRQ(ierr);
ierr = PetscFree(user);
CHKERRQ(ierr);
ierr = DMMGDestroy(dmmg);
CHKERRQ(ierr);
ierr = PetscFinalize();
CHKERRQ(ierr);
return result;
}
extern PetscErrorCode DefiantBlackOilComputePerfIndicesSyncPerfs(BlackOilReservoirSimulation* MySim)
{
PetscErrorCode ierr;
PetscInt mx, my, mz, xm, ym, zm, xs, ys, zs;
PetscInt i, rank;
/* Well handling variables */
PetscInt PerfIDMine, WellID;
PetscInt MyI, MyJ, MyK;
PetscScalar re;
/* Local values for variables*/
PetscScalar ***Localx1, ***Localx2, ***Localx3;
PetscScalar ***Localh1, ***Localh2, ***Localh3;
PetscScalar ***LocalSo, ***LocalSw, ***LocalSg;
PetscScalar ***LocalPo, ***LocalPw, ***LocalPg;
PetscScalar ***LocalMuo, ***LocalMuw, ***LocalMug;
PetscScalar ***LocalRhoo, ***LocalRhow, ***LocalRhog;
PetscScalar ***LocalK11, ***LocalK22, ***LocalK33;
PetscScalar ***LocalKro, ***LocalKrw, ***LocalKrg;
PetscScalar ***LocalBo, ***LocalBw, ***LocalBg;
PetscScalar ***LocalPcow, ***LocalPcog;
/* temporary array */
PetscScalar *InBuffer;
PetscFunctionBegin;
/* Allocate the In and out Buffers, we have 27 doubles at each perforation */
ierr = PetscMalloc(sizeof(PetscScalar)*27, &InBuffer);CHKERRQ(ierr);CHKMEMQ;
/* Get dimensions and extents of the local vectors */
ierr = DAGetInfo(MySim->SimDA, 0, &mx, &my, &mz, 0, 0, 0, 0, 0, 0, 0);CHKERRQ(ierr);
ierr = DAGetCorners(MySim->SimDA, &xs, &ys, &zs, &xm, &ym, &zm);CHKERRQ(ierr);
/* Grab the local coordiantes */
ierr = DAVecGetArray(MySim->SimDA, MySim->x1, &Localx1);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->x2, &Localx2);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->x3, &Localx3);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local geometry */
ierr = DAVecGetArray(MySim->SimDA, MySim->h1, &Localh1);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->h2, &Localh2);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->h3, &Localh3);CHKERRQ(ierr);CHKMEMQ;
/* Grab the Saturations */
ierr = DAVecGetArray(MySim->SimDA, MySim->So, &LocalSo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Sw, &LocalSw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Sg, &LocalSg);CHKERRQ(ierr);CHKMEMQ;
/* Grab the Pressures */
ierr = DAVecGetArray(MySim->SimDA, MySim->Po, &LocalPo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Pw, &LocalPw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Pg, &LocalPg);CHKERRQ(ierr);CHKMEMQ;
/* Grab the Densities */
ierr = DAVecGetArray(MySim->SimDA, MySim->Rhoo, &LocalRhoo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Rhow, &LocalRhow);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Rhog, &LocalRhog);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local viscosities*/
ierr = DAVecGetArray(MySim->SimDA, MySim->Muo, &LocalMuo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Muw, &LocalMuw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Mug, &LocalMug);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local permeabilities */
ierr = DAVecGetArray(MySim->SimDA, MySim->K11, &LocalK11);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->K22, &LocalK22);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->K33, &LocalK33);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local permeabilities */
ierr = DAVecGetArray(MySim->SimDA, MySim->Kro, &LocalKro);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Krw, &LocalKrw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Krg, &LocalKrg);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local capillary pressure */
ierr = DAVecGetArray(MySim->SimDA, MySim->Pcow, &LocalPcow);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Pcog, &LocalPcog);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local data for volume factors at the cell centers */
ierr = DAVecGetArray(MySim->SimDA, MySim->Bo, &LocalBo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Bw, &LocalBw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Bg, &LocalBg);CHKERRQ(ierr);CHKMEMQ;
/* Get the current rank */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);CHKMEMQ;
/* Now we sync all the flow rates and BHP's across all perforations */
/* At the end of this loop, all information across all wells is uniform and up-to-date */
for (WellID = 0; WellID < MySim->NumberOfWells; WellID++) {
for (PerfIDMine = 0; PerfIDMine
< (MySim->Wells[WellID]).NumberOfPerforations; PerfIDMine++) {
MyI = (MySim->Wells[WellID]).Perforations[PerfIDMine].I;CHKMEMQ;
MyJ = (MySim->Wells[WellID]).Perforations[PerfIDMine].J;CHKMEMQ;
MyK = (MySim->Wells[WellID]).Perforations[PerfIDMine].K;CHKMEMQ;
if (MyI >= xs && MyI < xs+xm && MyJ >= ys && MyJ < ys+ym && MyK >= zs && MyK < zs+zm)
{
MySim->Wells[WellID].Perforations[PerfIDMine].So = LocalSo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Sw = LocalSw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Sg = LocalSg[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Po = LocalPo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Pw = LocalPw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Pg = LocalPg[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Kro = LocalKro[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Krw = LocalKrw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Krg = LocalKrg[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Muo = LocalMuo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Muw = LocalMuw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Mug = LocalMug[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhoo = LocalRhoo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhow = LocalRhow[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhog = LocalRhog[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Pcow = LocalPcow[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Pcog = LocalPcog[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Bo = LocalBo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Bw = LocalBw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Bg = LocalBg[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].x1 = Localx1[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].x2 = Localx2[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].x3 = Localx3[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].h1 = Localh1[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].h2 = Localh2[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].h3 = Localh3[MyK][MyJ][MyI];CHKMEMQ;
/* we also compute the well indices */
if ((MySim->Wells[WellID]).Perforations[PerfIDMine].Orientation
== PERF_ORIENTATION_X1X1) {
re = 0.14 / 0.5 * sqrt((sqrt(LocalK33[MyK][MyJ][MyI]
/ LocalK22[MyK][MyJ][MyI]) * Localh2[MyK][MyJ][MyI]
* Localh2[MyK][MyJ][MyI] + sqrt(LocalK22[MyK][MyJ][MyI]
/ LocalK33[MyK][MyJ][MyI]) * Localh3[MyK][MyJ][MyI]
* Localh3[MyK][MyJ][MyI])) / (pow(LocalK33[MyK][MyJ][MyI]
/ LocalK22[MyK][MyJ][MyI], 0.25) + pow(LocalK22[MyK][MyJ][MyI]
/ LocalK33[MyK][MyJ][MyI], 0.25));
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex = 2.0 * PI
* Localh1[MyK][MyJ][MyI] * sqrt(LocalK11[MyK][MyJ][MyI]
* LocalK33[MyK][MyJ][MyI]) / (log(re
/ MySim->Wells[WellID].Perforations[PerfIDMine].Rw)
+ MySim->Wells[WellID].Perforations[PerfIDMine].S);
} else if ((MySim->Wells[WellID]).Perforations[PerfIDMine].Orientation
== PERF_ORIENTATION_X2X2) {
re = 0.14 / 0.5 * sqrt((sqrt(LocalK33[MyK][MyJ][MyI]
/ LocalK11[MyK][MyJ][MyI]) * Localh1[MyK][MyJ][MyI]
* Localh1[MyK][MyJ][MyI] + sqrt(LocalK11[MyK][MyJ][MyI]
/ LocalK33[MyK][MyJ][MyI]) * Localh3[MyK][MyJ][MyI]
* Localh3[MyK][MyJ][MyI])) / (pow(LocalK33[MyK][MyJ][MyI]
/ LocalK11[MyK][MyJ][MyI], 0.25) + pow(LocalK11[MyK][MyJ][MyI]
/ LocalK33[MyK][MyJ][MyI], 0.25));
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex = 2.0 * PI
* Localh2[MyK][MyJ][MyI] * sqrt(LocalK11[MyK][MyJ][MyI]
* LocalK33[MyK][MyJ][MyI]) / (log(re
/ MySim->Wells[WellID].Perforations[PerfIDMine].Rw)
+ MySim->Wells[WellID].Perforations[PerfIDMine].S);
} else if ((MySim->Wells[WellID]).Perforations[PerfIDMine].Orientation
== PERF_ORIENTATION_X3X3) {
re = 0.14 / 0.5 * sqrt((sqrt(LocalK22[MyK][MyJ][MyI]
/ LocalK11[MyK][MyJ][MyI]) * Localh1[MyK][MyJ][MyI]
* Localh1[MyK][MyJ][MyI] + sqrt(LocalK11[MyK][MyJ][MyI]
/ LocalK22[MyK][MyJ][MyI]) * Localh2[MyK][MyJ][MyI]
* Localh2[MyK][MyJ][MyI])) / (pow(LocalK22[MyK][MyJ][MyI]
/ LocalK11[MyK][MyJ][MyI], 0.25) + pow(LocalK11[MyK][MyJ][MyI]
/ LocalK22[MyK][MyJ][MyI], 0.25));
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex = 2.0 * PI
* Localh3[MyK][MyJ][MyI] * sqrt(LocalK11[MyK][MyJ][MyI]
* LocalK22[MyK][MyJ][MyI]) / (log(re
/ MySim->Wells[WellID].Perforations[PerfIDMine].Rw)
+ MySim->Wells[WellID].Perforations[PerfIDMine].S);
}
}
i = 0;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Po;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Pw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Pg;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].So;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Sw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Sg;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Rhoo;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Rhow;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Rhog;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Bo;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Bw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Bg;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Kro;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Krw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Krg;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Muo;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Muw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Mug;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].x1;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].x2;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].x3;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].h1;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].h2;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].h3;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Pcow;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Pcog;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex;
/* Sync properties at the perforation locations */
ierr = MPI_Bcast(InBuffer, 27, MPI_DOUBLE, MySim->Wells[WellID].Perforations[PerfIDMine].OwnerRank, PETSC_COMM_WORLD);CHKERRQ(ierr);CHKMEMQ;
/* and now unpack the data */
i = 0;
MySim->Wells[WellID].Perforations[PerfIDMine].Po = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Pw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Pg = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].So = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Sw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Sg = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhoo = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhow = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhog = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Bo = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Bw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Bg = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Kro = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Krw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Krg = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Muo = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Muw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Mug = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].x1 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].x2 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].x3 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].h1 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].h2 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].h3 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Pcow = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Pcog = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex = InBuffer[i];
PetscPrintf(PETSC_COMM_WORLD, "\n At well: %d Value of WellIndex is:%g", WellID,MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex);
}
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
DMMG *dmmg_comp; /* multilevel grid structure */
AppCtx user; /* user-defined work context */
PetscInt mx,my,its,max_its,i;
PetscErrorCode ierr;
MPI_Comm comm;
SNES snes;
DA da1,da2;
DMComposite pack;
DMMG *dmmg1,*dmmg2;
PetscTruth SolveSubPhysics=PETSC_FALSE,GaussSeidel=PETSC_TRUE,Jacobi=PETSC_FALSE;
Vec X1,X1_local,X2,X2_local;
PetscViewer viewer;
PetscInitialize(&argc,&argv,(char *)0,help);
comm = PETSC_COMM_WORLD;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create user context, set problem data, create vector data structures.
Also, compute the initial guess.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Setup Physics 1:
- Lap(U) - Grad_y(Omega) = 0
- Lap(V) + Grad_x(Omega) = 0
- Lap(Omega) + Div([U*Omega,V*Omega]) - GR*Grad_x(T) = 0
where T is given by the given x.temp
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DACreate2d(comm,DA_NONPERIODIC,DA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,0,0,&da1);CHKERRQ(ierr);
ierr = DASetFieldName(da1,0,"x-velocity");CHKERRQ(ierr);
ierr = DASetFieldName(da1,1,"y-velocity");CHKERRQ(ierr);
ierr = DASetFieldName(da1,2,"Omega");CHKERRQ(ierr);
/* Create the solver object and attach the grid/physics info */
ierr = DMMGCreate(comm,1,&user,&dmmg1);CHKERRQ(ierr);
ierr = DMMGSetDM(dmmg1,(DM)da1);CHKERRQ(ierr);
ierr = DMMGSetISColoringType(dmmg1,IS_COLORING_GLOBAL);CHKERRQ(ierr);
ierr = DMMGSetInitialGuess(dmmg1,FormInitialGuess1);CHKERRQ(ierr);
ierr = DMMGSetSNES(dmmg1,FormFunction1,0);CHKERRQ(ierr);
ierr = DMMGSetFromOptions(dmmg1);CHKERRQ(ierr);
/* Set problem parameters (velocity of lid, prandtl, and grashof numbers) */
ierr = DAGetInfo(da1,PETSC_NULL,&mx,&my,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
user.lidvelocity = 1.0/(mx*my);
user.prandtl = 1.0;
user.grashof = 1000.0;
ierr = PetscOptionsGetReal(PETSC_NULL,"-lidvelocity",&user.lidvelocity,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(PETSC_NULL,"-prandtl",&user.prandtl,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(PETSC_NULL,"-grashof",&user.grashof,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-solvesubphysics",&SolveSubPhysics);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-Jacobi",&Jacobi);CHKERRQ(ierr);
if (Jacobi) GaussSeidel=PETSC_FALSE;
ierr = PetscPrintf(comm,"grashof: %g, ",user.grashof);CHKERRQ(ierr);
if (GaussSeidel){
ierr = PetscPrintf(comm,"use Block Gauss-Seidel\n");CHKERRQ(ierr);
} else {
ierr = PetscPrintf(comm,"use Block Jacobi\n");CHKERRQ(ierr);
}
ierr = PetscPrintf(comm,"===========================================\n");CHKERRQ(ierr);
/* Solve the nonlinear system 1 */
if (SolveSubPhysics){
ierr = DMMGSolve(dmmg1);CHKERRQ(ierr);
snes = DMMGGetSNES(dmmg1);
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Physics 1: Number of Newton iterations = %D\n\n", its);CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Setup Physics 2:
- Lap(T) + PR*Div([U*T,V*T]) = 0
where U and V are given by the given x.u and x.v
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DACreate2d(comm,DA_NONPERIODIC,DA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0,&da2);CHKERRQ(ierr);
ierr = DASetFieldName(da2,0,"temperature");CHKERRQ(ierr);
/* Create the solver object and attach the grid/physics info */
ierr = DMMGCreate(comm,1,&user,&dmmg2);CHKERRQ(ierr);
ierr = DMMGSetDM(dmmg2,(DM)da2);CHKERRQ(ierr);
ierr = DMMGSetISColoringType(dmmg2,IS_COLORING_GLOBAL);CHKERRQ(ierr);
ierr = DMMGSetInitialGuess(dmmg2,FormInitialGuess2);CHKERRQ(ierr);
ierr = DMMGSetSNES(dmmg2,FormFunction2,0);CHKERRQ(ierr);
ierr = DMMGSetFromOptions(dmmg2);CHKERRQ(ierr);
/* Solve the nonlinear system 2 */
if (SolveSubPhysics){
ierr = DMMGSolve(dmmg2);CHKERRQ(ierr);
snes = DMMGGetSNES(dmmg2);
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Physics 2: Number of Newton iterations = %D\n\n", its);CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve system 1 and 2 iteratively
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DACreateLocalVector(da1,&X1_local);CHKERRQ(ierr);
ierr = DACreateLocalVector(da2,&X2_local);CHKERRQ(ierr);
/* Only 1 snes iteration is allowed for each subphysics */
/*
snes = DMMGGetSNES(dmmg1);
ierr = SNESSetTolerances(snes,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,1,PETSC_DEFAULT);CHKERRQ(ierr);
snes = DMMGGetSNES(dmmg2);
ierr = SNESSetTolerances(snes,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,1,PETSC_DEFAULT);CHKERRQ(ierr);
*/
max_its = 5;
ierr = PetscOptionsGetInt(PETSC_NULL,"-mp_max_it",&max_its,PETSC_NULL);CHKERRQ(ierr);
user.nsolve = 0;
for (i=0; i<max_its; i++){
ierr = PetscPrintf(comm,"\nIterative nsolve %D ...\n", user.nsolve);CHKERRQ(ierr);
if (!GaussSeidel){
/* get the ghosted X1_local for Physics 2 */
X1 = DMMGGetx(dmmg1); //Jacobian
if (i){ierr = DAVecRestoreArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr);}
ierr = DAGlobalToLocalBegin(da1,X1,INSERT_VALUES,X1_local);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da1,X1,INSERT_VALUES,X1_local);CHKERRQ(ierr);
ierr = DAVecGetArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr);
}
ierr = DMMGSolve(dmmg1);CHKERRQ(ierr);
snes = DMMGGetSNES(dmmg1);
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
if (GaussSeidel){
/* get the ghosted X1_local for Physics 2 */
X1 = DMMGGetx(dmmg1);
if (i){ierr = DAVecRestoreArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr);}
ierr = DAGlobalToLocalBegin(da1,X1,INSERT_VALUES,X1_local);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da1,X1,INSERT_VALUES,X1_local);CHKERRQ(ierr);
ierr = DAVecGetArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr);
}
ierr = PetscPrintf(comm," Iterative physics 1: Number of Newton iterations = %D\n", its);CHKERRQ(ierr);
user.nsolve++;
ierr = DMMGSolve(dmmg2);CHKERRQ(ierr);
snes = DMMGGetSNES(dmmg2);
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
/* get the ghosted X2_local for Physics 1 */
X2 = DMMGGetx(dmmg2);
if (i){ierr = DAVecRestoreArray(da2,X2_local,(Field2 **)&user.x2);CHKERRQ(ierr);}
ierr = DAGlobalToLocalBegin(da2,X2,INSERT_VALUES,X2_local);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da2,X2,INSERT_VALUES,X2_local);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2_local,(Field2 **)&user.x2);CHKERRQ(ierr);
ierr = PetscPrintf(comm," Iterative physics 2: Number of Newton iterations = %D\n", its);CHKERRQ(ierr);
//user.nsolve++;
}
ierr = DAVecRestoreArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2_local,(Field2 **)&user.x2);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the DMComposite object to manage the two grids/physics.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscPrintf(comm," \n\n DMComposite iteration......\n");CHKERRQ(ierr);
ierr = DMCompositeCreate(comm,&pack);CHKERRQ(ierr);
ierr = DMCompositeAddDM(pack,(DM)da1);CHKERRQ(ierr);
ierr = DMCompositeAddDM(pack,(DM)da2);CHKERRQ(ierr);
/* Create the solver object and attach the grid/physics info */
ierr = DMMGCreate(comm,1,&user,&dmmg_comp);CHKERRQ(ierr);
ierr = DMMGSetDM(dmmg_comp,(DM)pack);CHKERRQ(ierr);
ierr = DMMGSetISColoringType(dmmg_comp,IS_COLORING_GLOBAL);CHKERRQ(ierr);
ierr = DMMGSetInitialGuess(dmmg_comp,FormInitialGuessComp);CHKERRQ(ierr);
ierr = DMMGSetSNES(dmmg_comp,FormFunctionComp,0);CHKERRQ(ierr);
ierr = DMMGSetFromOptions(dmmg_comp);CHKERRQ(ierr);
/* Solve the nonlinear system */
/* ierr = DMMGSolve(dmmg_comp);CHKERRQ(ierr);
snes = DMMGGetSNES(dmmg_comp);
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Composite Physics: Number of Newton iterations = %D\n\n", its);CHKERRQ(ierr);*/
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free spaces
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DMCompositeDestroy(pack);CHKERRQ(ierr);
ierr = DADestroy(da1);CHKERRQ(ierr);
ierr = DADestroy(da2);CHKERRQ(ierr);
ierr = DMMGDestroy(dmmg_comp);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(comm,"log.py",&viewer);CHKERRQ(ierr);
/* -log_summary */
ierr = PetscLogPrintSummaryToPy(comm,viewer);CHKERRQ(ierr);
/* -snes_view */
//snes = DMMGGetSNES(dmmg1);CHKERRQ(ierr);
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
ierr = DMMGDestroy(dmmg1);CHKERRQ(ierr);
ierr = DMMGDestroy(dmmg2);CHKERRQ(ierr);
ierr = VecDestroy(X1_local);CHKERRQ(ierr);
ierr = VecDestroy(X2_local);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
