PetscErrorCode FormFunction(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DA da2 = (DA)dmmg->dm;
DALocalInfo info2;
Field2 **x2,**f2;
Vec X2;
PetscFunctionBegin;
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DAGetLocalVector(da2,&X2);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da2,X,INSERT_VALUES,X2);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da2,X,INSERT_VALUES,X2);CHKERRQ(ierr);
/* Access the array inside of X1 */
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted so directly access the memory locations in F */
ierr = DAVecGetArray(da2,F,(void**)&f2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormFunctionLocal2(&info2,0,x2,f2,(void**)user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,F,(void**)&f2);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DARestoreLocalVector(da2,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormInitialGuessComp -
Forms the initial guess for the composite model
Unwraps the global solution vector and passes its local pieces into the user functions
*/
PetscErrorCode FormInitialGuessComp(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DMComposite dm = (DMComposite)dmmg->dm;
Vec X1,X2;
Field1 **x1;
Field2 **x2;
DALocalInfo info1,info2;
DA da1,da2;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2);CHKERRQ(ierr);
/* Access the subvectors in X */
ierr = DMCompositeGetAccess(dm,X,&X1,&X2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal1(&info1,x1);CHKERRQ(ierr);
ierr = FormInitialGuessLocal2(&info2,x2,user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,X,&X1,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Compute the values of the surface mass balance, ice hardness, and sliding coeff. */
static PetscErrorCode FillDistributedParams(AppCtx *user)
{
PetscErrorCode ierr;
PetscInt i,Mx=user->Mx;
PetscScalar x, dum1, dum2, dum3, dum4, dum5, *M, *Bstag, *beta;
PetscFunctionBegin;
/* Compute regular grid exact soln and staggered-grid thickness over the
locally-owned part of the grid */
ierr = DAVecGetArray(user->scalarda,user->M,&M);CHKERRQ(ierr);
ierr = DAVecGetArray(user->scalarda,user->Bstag,&Bstag);CHKERRQ(ierr);
ierr = DAVecGetArray(user->scalarda,user->beta,&beta);CHKERRQ(ierr);
for (i = user->xs; i < user->xs + user->xm; i++) {
x = user->dx * (PetscReal)i; /* = x_i = distance from dome; regular grid point */
ierr = exactN(x, &dum1, &dum2, &dum3, &(M[i]), &dum4, &(beta[i])); CHKERRQ(ierr);
x = x + (user->dx/2.0); /* = x_{i+1/2}; staggered grid point */
if (i < Mx-1) {
ierr = exactN(x, &dum1, &dum2, &dum3, &dum4, &(Bstag[i]), &dum5); CHKERRQ(ierr);
} else {
Bstag[i] = -9999.9999; /* never accessed, and we don't have the value anyway */
}
}
ierr = DAVecRestoreArray(user->scalarda,user->M,&M);CHKERRQ(ierr);
ierr = DAVecRestoreArray(user->scalarda,user->Bstag,&Bstag);CHKERRQ(ierr);
ierr = DAVecRestoreArray(user->scalarda,user->beta,&beta);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void CopyBoundary(State *BHD, Vec gfrom, Vec gto, real zpad)
{
PData PD;
DA da;
int x[3], n[3], i[3], ic[3], dim, k, N[3], border;
PetscScalar ***gfrom_vec, ***gto_vec;
PD = BHD->PD;
da = BHD->da;
FOR_DIM
N[dim] = PD->N[dim];
VecSet(gto, 0.0);
border = (int) ceil( ((PD->interval[1]-PD->interval[0])-(2.*zpad))/PD->h[0]/2. );
/* Get local portion of the grid */
DAGetCorners(da, &(x[0]), &(x[1]), &(x[2]), &(n[0]), &(n[1]), &(n[2]));
DAVecGetArray(da, gfrom, &gfrom_vec);
DAVecGetArray(da, gto, >o_vec);
/* FOR_DIM */
/* { */
/* ic[0]=dim; */
/* ic[1]=(dim+1)%3; */
/* ic[2]=(dim+2)%3; */
/* i[ic[0]] = 0; */
/* if( x[ic[0]] == 0 ) */
/* for( i[ic[1]]=x[ic[1]]; i[ic[1]]<x[ic[1]]+n[ic[1]]; i[ic[1]]++) */
/* for( i[ic[2]]=x[ic[2]]; i[ic[2]]<x[ic[2]]+n[ic[2]]; i[ic[2]]++) */
/* gto_vec[i[2]][i[1]][i[0]] = gfrom_vec[i[2]][i[1]][i[0]]; */
/* i[ic[0]] = N[ic[0]]-1; */
/* if( x[ic[0]]+n[ic[0]] == N[ic[0]] ) */
/* for( i[ic[1]]=x[ic[1]]; i[ic[1]]<x[ic[1]]+n[ic[1]]; i[ic[1]]++) */
/* for( i[ic[2]]=x[ic[2]]; i[ic[2]]<x[ic[2]]+n[ic[2]]; i[ic[2]]++) */
/* gto_vec[i[2]][i[1]][i[0]] = gfrom_vec[i[2]][i[1]][i[0]]; */
/* } */
/* loop over local portion of grid */
for(i[2]=x[2]; i[2]<x[2]+n[2]; i[2]++)
for(i[1]=x[1]; i[1]<x[1]+n[1]; i[1]++)
for(i[0]=x[0]; i[0]<x[0]+n[0]; i[0]++)
{
if( ( i[0]<=border+1 || i[0]>=N[0]-1-border) ||
( i[1]<=border+1 || i[1]>=N[1]-1-border) ||
( i[2]<=border+1 || i[2]>=N[2]-1-border) )
gto_vec[i[2]][i[1]][i[0]] = gfrom_vec[i[2]][i[1]][i[0]];
}
DAVecRestoreArray(da, gfrom, &gfrom_vec);
DAVecRestoreArray(da, gto, >o_vec);
}
PetscErrorCode equation(SNES snes, Vec soln_vec, Vec fun_vec, void* ptr)
{
//data structure will be contained in ptr
Data* data = (Data*) ptr;
Params P = data->P;
DA da = data->da;
DALocalInfo info;
PetscFunctionBegin;
DAGetLocalInfo(da, &info);
Vec sv_local; //need local vector containing ghost point information
DACreateLocalVector(da, &sv_local);
DAGlobalToLocalBegin(da, soln_vec, INSERT_VALUES, sv_local);
DAGlobalToLocalEnd(da, soln_vec, INSERT_VALUES, sv_local);
//to evaluate fuction, the sv_local soln_vec need to be converted into Field arrays
Field **X;
Field **F;
DAVecGetArray(da, sv_local, &X);
DAVecGetArray(da, fun_vec, &F);
//since sv_local is local vector, X will have access to ghost points
PetscInt i, j, Ny = P.Ny;
PetscScalar hx = P.hx; hy = P.hy; x, y;
for(i=info.xs; i<info.xs + info.xm; i++){
for(j=info.ys; j<info.ys+info.ym; j++){
x = i *hx;
y = j *hy;
if(j==0) {
F[j][i].u = X[j][i].u - 0.5*(sin(2*PETSC_PI*x) + 1);
F[j][i].v = X[j][i].v ;
}
else if (j== Ny-1){
F[j][i].u = X[j][i].u;
F[j][i].v = X[j][i].v - 0.5*(sin(2*PETSC_PI*x) + 1);
}
else {
F[j][i].u = lap(u) + X[j][i].u * X[j][i].v ;
F[j][i].v = lap(v) - X[j][i].u * X[j][i].v ;
}
}
}
DAVecRestoreArray(da, fun_vec, &F);
DAVecRestoreArray(da, sv_local, &X);
VecDestory(sv_local);
PetscFuntionReturn(0);
}
/*
FormInitialGuess - Unwraps the global solution vector and passes its local pieces into the user function
*/
PetscErrorCode FormInitialGuess(DMMG dmmg,Vec X)
{
DMComposite dm = (DMComposite)dmmg->dm;
DALocalInfo info1,info2,info3;
DA da1,da2,da3;
FluidField *x1;
PetscScalar **x2;
FuelField **x3;
Vec X1,X2,X3;
PetscErrorCode ierr;
AppCtx *app = (AppCtx*)dmmg->user;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2,&da3);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da3,&info3);
CHKERRQ(ierr);
/* Access the three subvectors in X */
ierr = DMCompositeGetAccess(dm,X,&X1,&X2,&X3);
CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);
CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);
CHKERRQ(ierr);
ierr = DAVecGetArray(da3,X3,(void**)&x3);
CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocalFluid(app,&info1,x1);
CHKERRQ(ierr);
ierr = FormInitialGuessLocalThermal(app,&info2,x2);
CHKERRQ(ierr);
ierr = FormInitialGuessLocalFuel(app,&info3,x3);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da3,X3,(void**)&x3);
CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,X,&X1,&X2,&X3);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeFunction(SNES snes,Vec x,Vec f,void *ctx) {
PetscInt i,Mx,xs,xm; PetscScalar *xx,*ff,hx; DA da = (DA) ctx; Vec xlocal;
DAGetInfo(da,PETSC_IGNORE,&Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
hx = 1.0/(PetscReal)(Mx-1);
DAGetLocalVector(da,&xlocal);DAGlobalToLocalBegin(da,x,INSERT_VALUES,xlocal); DAGlobalToLocalEnd(da,x,INSERT_VALUES,xlocal);
DAVecGetArray(da,xlocal,&xx); DAVecGetArray(da,f,&ff);
DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);
for (i=xs; i<xs+xm; i++) {
if (i == 0 || i == Mx-1) ff[i] = xx[i]/hx;
else ff[i] = (2.0*xx[i] - xx[i-1] - xx[i+1])/hx - hx*PetscExpScalar(xx[i]);
}
DAVecRestoreArray(da,xlocal,&xx); DARestoreLocalVector(da,&xlocal);DAVecRestoreArray(da,f,&ff);
return 0;}
/*
FormFunctionComp - Unwraps the input vector and passes its local ghosted pieces into the user function
*/
PetscErrorCode FormFunctionComp(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DMComposite dm = (DMComposite)dmmg->dm;
DALocalInfo info1,info2;
DA da1,da2;
Field1 **x1,**f1;
Field2 **x2,**f2;
Vec X1,X2,F1,F2;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DMCompositeGetLocalVectors(dm,&X1,&X2);CHKERRQ(ierr);
ierr = DMCompositeScatter(dm,X,X1,X2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted and directly access the memory locations in F */
ierr = DMCompositeGetAccess(dm,F,&F1,&F2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of F */
ierr = DAVecGetArray(da1,F1,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,F2,(void**)&f2);CHKERRQ(ierr);
/* Evaluate local user provided function */
user->x2 = x2; /* used by FormFunctionLocal1() */
ierr = FormFunctionLocal1(&info1,x1,f1,(void**)user);CHKERRQ(ierr);
user->x1 = x1; /* used by FormFunctionLocal2() */
ierr = FormFunctionLocal2(&info2,x2,f2,(void**)user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,F1,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,F2,(void**)&f2);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,F,&F1,&F2);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DMCompositeRestoreLocalVectors(dm,&X1,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
PostCheck - Optional user-defined routine that checks the validity of
candidate steps of a line search method. Set by SNESLineSearchSetPostCheck().
Input Parameters:
snes - the SNES context
ctx - optional user-defined context for private data for the
monitor routine, as set by SNESLineSearchSetPostCheck()
xcurrent - current solution
y - search direction and length
x - the new candidate iterate
Output Parameters:
y - proposed step (search direction and length) (possibly changed)
x - current iterate (possibly modified)
*/
PetscErrorCode PostCheck(SNES snes,Vec xcurrent,Vec y,Vec x,void *ctx,PetscTruth *changed_y,PetscTruth *changed_x)
{
PetscErrorCode ierr;
PetscInt i,iter,xs,xm;
StepCheckCtx *check = (StepCheckCtx*) ctx;
ApplicationCtx *user = check->user;
PetscScalar *xa,*xa_last,tmp;
PetscReal rdiff;
DA da;
PetscFunctionBegin;
*changed_x = PETSC_FALSE;
*changed_y = PETSC_FALSE;
ierr = SNESGetIterationNumber(snes,&iter);CHKERRQ(ierr);
/* iteration 1 indicates we are working on the second iteration */
if (iter > 0) {
da = user->da;
ierr = PetscPrintf(PETSC_COMM_WORLD,"Checking candidate step at iteration %D with tolerance %G\n",iter,check->tolerance);CHKERRQ(ierr);
/* Access local array data */
ierr = DAVecGetArray(da,check->last_step,&xa_last);CHKERRQ(ierr);
ierr = DAVecGetArray(da,x,&xa);CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
/*
If we fail the user-defined check for validity of the candidate iterate,
then modify the iterate as we like. (Note that the particular modification
below is intended simply to demonstrate how to manipulate this data, not
as a meaningful or appropriate choice.)
*/
for (i=xs; i<xs+xm; i++) {
if (!PetscAbsScalar(xa[i])) rdiff = 2*check->tolerance; else rdiff = PetscAbsScalar((xa[i] - xa_last[i])/xa[i]);
if (rdiff > check->tolerance) {
tmp = xa[i];
xa[i] = .5*(xa[i] + xa_last[i]);
*changed_x = PETSC_TRUE;
ierr = PetscPrintf(PETSC_COMM_WORLD," Altering entry %D: x=%G, x_last=%G, diff=%G, x_new=%G\n",
i,PetscAbsScalar(tmp),PetscAbsScalar(xa_last[i]),rdiff,PetscAbsScalar(xa[i]));CHKERRQ(ierr);
}
}
ierr = DAVecRestoreArray(da,check->last_step,&xa_last);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da,x,&xa);CHKERRQ(ierr);
}
ierr = VecCopy(x,check->last_step);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FluidFieldDivergence_3D( DALocalInfo info,
Coor d, Vec US, Vec VS, Vec WS, Vec DIV )
{
int i,j,k;
PetscReal xe, xs, ye, ys, ze, zs;
PetscReal ***u, ***v, ***w, ***div;
PetscErrorCode ierr;
PetscFunctionBegin;
xs = info.xs;
xs = xs == 0 ? 2 : xs;
xe = info.xs+info.xm;
xe = xe == info.mx ? info.mx-3 : xe;
ys = info.ys;
ys = ys == 0 ? 2 : ys;
ye = info.ys+info.ym;
ye = ye == info.my ? info.my-3 : ye;
zs = info.zs;
zs = zs == 0 ? 2 : zs;
ze = info.zs+info.zm;
ze = ze == info.mz ? info.mz-3 : ze;
ierr = DAVecGetArray(info.da,US,&u); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,VS,&v); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,WS,&w); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,DIV,&div); CHKERRQ(ierr);
for( k = zs; k < ze; ++k)
{
for (j = ys; j < ye; ++j)
{
for (i = xs; i < xe; ++i)
{
div[k][j][i] = (u[k][j][i+1] - u[k][j][i]) / d.x +
(v[k][j+1][i] - v[k][j][i]) / d.y +
(w[k+1][j][i] - w[k][j][i]) / d.z;
}
}
}
ierr = DAVecRestoreArray(info.da,US,&u); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,VS,&v); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,WS,&w); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,DIV,&div); CHKERRQ(ierr);
PetscLogFlops(8*(xe-xs)*(ye-ys)*(ze-zs));
PetscFunctionReturn(0);
}
PetscErrorCode testCreate2D()
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 1453, d2 = 1451;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal **v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym;
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[j][i] = 1.*i + d1 * j;
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Updated local portion with DAVec\n");
PetscBarrier(0);
{
double *da_ptr;
VecGetArray(vec, &da_ptr);
double *ptr;
int low[2],hi[2],ld;
NGA_Distribution(ga,GA_Nodeid(),low,hi);
NGA_Access(ga,low,hi,&ptr,&ld);
printf("[%d] ga:%p\tda:%p\tdiff:%p\n", GA_Nodeid(), ptr, da_ptr, (ptr-da_ptr) );
NGA_Release_update(ga,low,hi);
}
int lo[2],ld;
double val;
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = j;
lo[1] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j - val) > .1 )
printf(".");
// printf("[%d] (%3.0f,%3.0f)\n", GA_Nodeid(), 1.*i + d1*j, val);
}
}
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
/*
MSA_InitialPoint - Calculates the initial guess in one of three ways.
Input Parameters:
. user - user-defined application context
. X - vector for initial guess
Output Parameters:
. X - newly computed initial guess
*/
static int MSA_InitialPoint(AppCtx * user, Vec X)
{
int info;
PetscInt start2=-1,i,j;
PetscReal start1=0;
PetscTruth flg1,flg2;
info = PetscOptionsGetReal(PETSC_NULL,"-start",&start1,&flg1); CHKERRQ(info);
info = PetscOptionsGetInt(PETSC_NULL,"-random",&start2,&flg2); CHKERRQ(info);
if (flg1){ /* The zero vector is reasonable */
info = VecSet(X, start1); CHKERRQ(info);
} else if (flg2 && start2>0){ /* Try a random start between -0.5 and 0.5 */
PetscRandom rctx; PetscScalar np5=-0.5;
info = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);
CHKERRQ(info);
for (i=0; i<start2; i++){
info = VecSetRandom(X, rctx); CHKERRQ(info);
}
info = PetscRandomDestroy(rctx); CHKERRQ(info);
info = VecShift(X, np5); CHKERRQ(info);
} else { /* Take an average of the boundary conditions */
PetscInt xs,xm,ys,ym;
PetscInt mx=user->mx,my=user->my;
PetscScalar **x;
/* Get local mesh boundaries */
info = DAGetCorners(user->da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL); CHKERRQ(info);
/* Get pointers to vector data */
info = DAVecGetArray(user->da,X,(void**)&x);
/* Perform local computations */
for (j=ys; j<ys+ym; j++){
for (i=xs; i< xs+xm; i++){
x[j][i] = ( ((j+1)*user->bottom[i-xs+1]+(my-j+1)*user->top[i-xs+1])/(my+2)+
((i+1)*user->left[j-ys+1]+(mx-i+1)*user->right[j-ys+1])/(mx+2))/2.0;
}
}
/* Restore vectors */
info = DAVecRestoreArray(user->da,X,(void**)&x); CHKERRQ(info);
info = PetscLogFlops(9*xm*ym); CHKERRQ(info);
}
return 0;
}
PetscErrorCode FormFunction1(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DA da1 = (DA)dmmg->dm;
DALocalInfo info1;
Field1 **x1,**f1;
Vec X1;
Field2 **x2;
PetscFunctionBegin;
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DAGetLocalVector(da1,&X1);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da1,X,INSERT_VALUES,X1);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da1,X,INSERT_VALUES,X1);CHKERRQ(ierr);
/* Access the arrays inside X1 */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted so directly access the memory locations in F */
ierr = DAVecGetArray(da1,F,(void**)&f1);CHKERRQ(ierr);
/* Evaluate local user provided function */
if (user->nsolve){
x2 = user->x2;
ierr = FormFunctionLocal1(&info1,x1,x2,f1,(void**)user);CHKERRQ(ierr);
} else {
ierr = FormFunctionLocal1(&info1,x1,0,f1,(void**)user);CHKERRQ(ierr);
}
ierr = DAVecRestoreArray(da1,F,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DARestoreLocalVector(da1,&X1);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start %s\n", __FILE__); CHKERRQ(ierr);
MPI_Comm comm = PETSC_COMM_WORLD;
DA da;
Mat mat;
int MX = 100;
int dof = 3;
ierr = PetscPrintf(comm,"Started Assembly\n"); CHKERRQ(ierr);
ierr = MatMake(MX,dof,&da, &mat); CHKERRQ(ierr);
ierr = MatWrite("J",mat); CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Finished Assembly\n"); CHKERRQ(ierr);
Vec rhs, sol;
ierr = DACreateGlobalVector(da,&rhs); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&sol); CHKERRQ(ierr);
int i,j;
int xs,ys,xm,ym;
Field **array;
ierr = DAVecGetArray(da,rhs,&array); CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL); CHKERRQ(ierr);
for (j = xs; j < xm; ++j) {
for (i = ys; i < ym; ++i) {
if( 3*MX/4 > i && i > MX/4 &&
3*MX/4 > j && j > MX/4 )
{
array[j][i].u = 100.;
array[j][i].v = 100.;
}
}
}
ierr = DAVecRestoreArray(da,rhs,&array); CHKERRQ(ierr);
KSP ksp;
ierr = KSPCreate(comm,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY); CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,mat,mat,DIFFERENT_NONZERO_PATTERN); CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
//Split pressure from velocity
PC pc;
ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr);
ierr = PCFieldSplitSetFields(pc,2,(int[]){0,1}); CHKERRQ(ierr);
/* Put a not-unreasonable initial guess in Hu. */
static PetscErrorCode FillInitial(AppCtx *user, Vec *vHu)
{
PetscErrorCode ierr;
PetscInt i;
Node *Hu;
PetscFunctionBegin;
ierr = DAVecGetArray(user->da,*vHu,&Hu);CHKERRQ(ierr);
for (i = user->xs; i < user->xs + user->xm; i++) {
Hu[i].H = 1000.0;
Hu[i].u = 100.0 / user->secpera;
}
ierr = DAVecRestoreArray(user->da,*vHu,&Hu);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int FormBratuFunction(SNES snes, Vec v, Vec f, void* ctx)
{
UserBratuCtx* bratu = (UserBratuCtx *) ctx;
DA da = bratu->da;
double lambda = bratu->lambda;
double h = bratu->h;
Vec lv;
int i,j;
int lli, llj, ni, nj;
const double **varr;
double **fvarr;
DAGetCorners(da, &lli, &llj, 0, &ni, &nj, 0);
DAGetLocalVector(da, &lv);
DAGlobalToLocalBegin(da, v, INSERT_VALUES, lv);
DAGlobalToLocalEnd(da, v, INSERT_VALUES,lv);
DAVecGetArray(da, lv, (void**) &varr);
DAVecGetArray(da, f, (void**) &fvarr);
for(j=llj; j<llj+nj; j++){
for(i=lli; i<lli+ni; i++){
if(i==0 || j==0 ||i=bratu->n+1 || j==bratu->n+1){
fvarr[j][i] = 0.0;
}
else{
fvarr[j][i] = -(varr[j-1][i] + varr[j][i-1] + varr[j+1][i] + varr[j][i+1] - 4*varr[j][i])/(h*h) - lambda*exp(varr[j][i]);
}
}
}
DAVecRestoreArray(da, f, (void**) &fvarr);
DAVecRestoreArray(da, lv, (void**) &varr);
DARestoreLocalVector(da, &lv);
return 0;
}
PetscErrorCode FluidFieldDivergence_2D( DALocalInfo info, Coor d,
Vec US, Vec VS, Vec DIV )
{
int i,j;
PetscReal xe, xs, ye, ys;
PetscReal **u, **v, **div;
PetscErrorCode ierr;
PetscFunctionBegin;
xs = info.xs;
xs = xs == 0 ? 2 : xs;
xe = info.xs+info.xm;
xe = xe == info.mx ? info.mx-3 : xe;
ys = info.ys;
ys = ys == 0 ? 2 : ys;
ye = info.ys+info.ym;
ye = ye == info.my ? info.my-3 : ye;
ierr = DAVecGetArray(info.da,US,&u); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,VS,&v); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,DIV,&div); CHKERRQ(ierr);
for (j = ys; j < ye; ++j)
{
for (i = xs; i < xe; ++i)
{
div[j][i] = (u[j][i+1] - u[j][i]) / d.x +
(v[j+1][i] - v[j][i]) / d.y;
}
}
ierr = DAVecRestoreArray(info.da,US,&u); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,VS,&v); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,DIV,&div); CHKERRQ(ierr);
PetscLogFlops(5*(xe-xs)*(ye-ys));
PetscFunctionReturn(0);
}
/*
FormInitialGuessLocal - Forms initial approximation for this process
Input Parameters:
user - user-defined application context
X - vector (DA local vector)
Output Parameter:
X - vector with the local values set
*/
PetscErrorCode FormInitialGuessLocal(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
PetscReal grashof,dx;
Field **x;
grashof = user->grashof;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
dx = 1.0/(mx-1);
/*
Get local grid boundaries (for 2-dimensional DA):
xs, ys - starting grid indices (no ghost points)
xm, ym - widths of local grid (no ghost points)
*/
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
/*
Get a pointer to vector data.
- For default PETSc vectors, VecGetArray() returns a pointer to
the data array. Otherwise, the routine is implementation dependent.
- You MUST call VecResetArraystoreArray() when you no longer need access to
the array.
*/
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
/*
Compute initial guess over the locally owned part of the grid
Initial condition is motionless fluid and equilibrium temperature
U = V = Omega = 0.0; Temp[x_i, y_j] = x_i;
*/
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].u = 0.0;
x[j][i].v = 0.0;
x[j][i].omega = 0.0;
x[j][i].temp = (grashof>0)*i*dx;
}
}
/* Restore vector */
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
int main(int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start\n"); CHKERRQ(ierr);
int d1 = 5;
ierr = PetscOptionsSetValue("-da_grid_x","5"); CHKERRQ(ierr);
int d2 = 7;
ierr = PetscOptionsSetValue("-da_grid_y","7"); CHKERRQ(ierr);
FluidField f;
ierr = FluidFieldCreate(&f); CHKERRQ(ierr);
iCoor s = {d1,d2,0};
Grid g;
ierr = GridCreate(s,&g); CHKERRQ(ierr);
PetscReal *p;
VecGetArray(f->p, &p);
for (int i = 0; i < g->n.x*g->n.y; ++i)
{
g->v1[i] = i;
p[i] = i;
}
PetscReal **pp;
DAVecGetArray(f->da, f->p, &pp);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
printf( "%1.0f, %1.0f\t", g->v2[j][i], pp[j][i]);
}
printf("\n");
}
DAVecRestoreArray(f->da, f->p, &pp);
ierr = GridDestroy(g); CHKERRQ(ierr);
ierr = FluidFieldDestroy(f); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "End\n"); CHKERRQ(ierr);
ierr = PetscFinalize(); CHKERRQ(ierr);
return 0;
}
/* Compute the exact thickness and velocity on the regular grid. */
static PetscErrorCode FillExactSoln(AppCtx *user)
{
PetscErrorCode ierr;
PetscInt i;
PetscScalar x, dum1, dum2, dum3, dum4;
Node *Hu;
PetscFunctionBegin;
/* Compute regular grid exact soln and staggered-grid thickness over the
locally-owned part of the grid */
ierr = DAVecGetArray(user->da,user->Huexact,&Hu);CHKERRQ(ierr);
for (i = user->xs; i < user->xs + user->xm; i++) {
x = user->dx * (PetscReal)i; /* = x_i = distance from dome */
ierr = exactN(x, &(Hu[i].H), &dum1, &(Hu[i].u), &dum2, &dum3, &dum4); CHKERRQ(ierr);
}
ierr = DAVecRestoreArray(user->da,user->Huexact,&Hu);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int Initialize(DMMG *dmmg)
/* ------------------------------------------------------------------- */
{
AppCtx *user = (AppCtx*)dmmg[0]->user;
Parameter *param;
DA da;
PetscReal PI = 3.14159265358979323846;
PetscReal sigma,xc,zc;
PetscReal dx=user->grid->dx,dz=user->grid->dz;
int i,j,ierr,is,js,im,jm;
Field **x;
ierr = PetscBagGetData(user->bag,(void**)¶m);
CHKERRQ(ierr);
sigma=param->sigma;
xc=param->xctr;
zc=param->zctr;
/* Get the DA and grid */
da = (DA)(dmmg[0]->dm);
ierr = DAGetCorners(da,&is,&js,PETSC_NULL,&im,&jm,PETSC_NULL);
CHKERRQ(ierr);
ierr = DAVecGetArray(da,user->Xold,(void**)&x);
CHKERRQ(ierr);
for (j=js; j<js+jm; j++) {
for (i=is; i<is+im; i++) {
if (param->flow_type == SHEAR_CELL) {
x[j][i].u = -sin(PI*i*dx)*cos(PI*j*dz)/dx;
x[j][i].w = sin(PI*j*dz)*cos(PI*i*dx)/dz;
} else {
x[j][i].u = 0.0;
x[j][i].w = -1.0/dz;
}
x[j][i].phi = 100*exp(-0.5*((i*dx-xc)*(i*dx-xc)+(j*dz-zc)*(j*dz-zc))/sigma/sigma);
}
}
/* restore the grid to it's vector */
ierr = DAVecRestoreArray(da,user->Xold,(void**)&x);
CHKERRQ(ierr);
ierr = VecCopy(user->Xold, DMMGGetx(dmmg));
CHKERRQ(ierr);
return 0;
}
PetscErrorCode ComputeJacobian(SNES snes,Vec x,Mat *J,Mat *B,MatStructure *flag,void *ctx){
DA da = (DA) ctx; PetscInt i,Mx,xm,xs; PetscScalar hx,*xx; Vec xlocal;
DAGetInfo(da,PETSC_IGNORE,&Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
hx = 1.0/(PetscReal)(Mx-1);
DAGetLocalVector(da,&xlocal);DAGlobalToLocalBegin(da,x,INSERT_VALUES,xlocal); DAGlobalToLocalEnd(da,x,INSERT_VALUES,xlocal);
DAVecGetArray(da,xlocal,&xx);
DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);
for (i=xs; i<xs+xm; i++) {
if (i == 0 || i == Mx-1) { MatSetValue(*J,i,i,1.0/hx,INSERT_VALUES);}
else {
MatSetValue(*J,i,i-1,-1.0/hx,INSERT_VALUES);
MatSetValue(*J,i,i,2.0/hx - hx*PetscExpScalar(xx[i]),INSERT_VALUES);
MatSetValue(*J,i,i+1,-1.0/hx,INSERT_VALUES);
}
}
MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY); MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY); *flag = SAME_NONZERO_PATTERN;
DAVecRestoreArray(da,xlocal,&xx);DARestoreLocalVector(da,&xlocal);
return 0;}
PetscErrorCode FormInitialGuess(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DA da2 = (DA)dmmg->dm;
Field2 **x2;
DALocalInfo info2;
PetscFunctionBegin;
/* Access the arrays inside of X */
ierr = DAVecGetArray(da2,X,(void**)&x2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal2(&info2,x2,user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X,(void**)&x2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode doit(DA da,Vec global)
{
PetscErrorCode ierr;
PetscInt i,j,k,M,N,dof;
ierr = DAGetInfo(da,0,&M,&N,0,0,0,0,&dof,0,0,0);CHKERRQ(ierr);
{
struct {PetscScalar inside[dof];} **mystruct;
ierr = DAVecGetArray(da,global,(void*) &mystruct);
for ( i=0; i<N; i++) {
for ( j=0; j<M; j++) {
for ( k=0; k<dof; k++) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"%d %d %G\n",i,j,mystruct[i][j].inside[0]);CHKERRQ(ierr);
mystruct[i][j].inside[1] = 2.1;
}
}
}
ierr = DAVecRestoreArray(da,global,(void*) &mystruct);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/* Form initial guess for Physic 1 */
PetscErrorCode FormInitialGuessLocal1(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
Field1 **x;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].u = 0.0;
x[j][i].v = 0.0;
x[j][i].omega = 0.0;
}
}
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
/* For now, we assume that only the bottom boundary is changing! No I have upgraded it to a cooler sexier version! :-) */
Immersed *Immersed_create(MAC_grid *grid, char which_quantity) {
int N;
Immersed *new_immersed;
int NZ, NY, NX;
int ierr;
NX = grid->NX;
NY = grid->NY;
NZ = grid->NZ;
new_immersed = (Immersed *)malloc(sizeof(Immersed));
Memory_check_allocation(new_immersed);
/* Total number of the immersed nodes. It should be more or equal to NX */
/* TBC */
N = Immersed_count_immersed_nodes(grid, which_quantity);
PetscPrintf(PCW, "Found %d immeresed points in the domain on the current processor for the %c-quantity\n", N, which_quantity);
/* Maximum number of immersed nodes in the whole domain */
new_immersed->N = N;
/* Allocate the memory for each individual immersed node */
new_immersed->ib_nodes = (ImmersedNode *)calloc(N, sizeof(ImmersedNode));
Memory_check_allocation(new_immersed->ib_nodes);
/* Allocate memory for the global index of the current ib_node in the entire domain. If not an ib_node, the value is -1 */
/* This will help finding the (i,j,k) location of the immersed node */
ierr = DACreateGlobalVector(grid->DA_3D, &new_immersed->G_global_index); PETScErrAct(ierr);
ierr = DACreateLocalVector(grid->DA_3D, &new_immersed->L_global_index); PETScErrAct(ierr);
/* Pass the 3d int-pointer to the L_global_index. This will help save a huge amount of time */
ierr = DAVecGetArray(grid->DA_3D, new_immersed->L_global_index, (void ***)&new_immersed->global_index); PETScErrAct(ierr);
/* u, v, c (or p) */
new_immersed->quantity = which_quantity;
return new_immersed;
}
PetscErrorCode FormInitialGuess1(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DA da1 = (DA)dmmg->dm;
Field1 **x1;
DALocalInfo info1;
PetscFunctionBegin;
if (user->nsolve) PetscFunctionReturn(0);
printf(" FormInitialGuess1 ... user.nsolve %d\n",user->nsolve);
/* Access the array inside of X */
ierr = DAVecGetArray(da1,X,(void**)&x1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal1(&info1,x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X,(void**)&x1);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Form initial guess for Physic 2 */
PetscErrorCode FormInitialGuessLocal2(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
PetscReal grashof,dx;
Field2 **x;
grashof = user->grashof;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
dx = 1.0/(mx-1);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].temp = (grashof>0)*i*dx;
}
}
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
extern PetscErrorCode DefiantBlackOilComputePerfIndicesSyncPerfs(BlackOilReservoirSimulation* MySim)
{
PetscErrorCode ierr;
PetscInt mx, my, mz, xm, ym, zm, xs, ys, zs;
PetscInt i, rank;
/* Well handling variables */
PetscInt PerfIDMine, WellID;
PetscInt MyI, MyJ, MyK;
PetscScalar re;
/* Local values for variables*/
PetscScalar ***Localx1, ***Localx2, ***Localx3;
PetscScalar ***Localh1, ***Localh2, ***Localh3;
PetscScalar ***LocalSo, ***LocalSw, ***LocalSg;
PetscScalar ***LocalPo, ***LocalPw, ***LocalPg;
PetscScalar ***LocalMuo, ***LocalMuw, ***LocalMug;
PetscScalar ***LocalRhoo, ***LocalRhow, ***LocalRhog;
PetscScalar ***LocalK11, ***LocalK22, ***LocalK33;
PetscScalar ***LocalKro, ***LocalKrw, ***LocalKrg;
PetscScalar ***LocalBo, ***LocalBw, ***LocalBg;
PetscScalar ***LocalPcow, ***LocalPcog;
/* temporary array */
PetscScalar *InBuffer;
PetscFunctionBegin;
/* Allocate the In and out Buffers, we have 27 doubles at each perforation */
ierr = PetscMalloc(sizeof(PetscScalar)*27, &InBuffer);CHKERRQ(ierr);CHKMEMQ;
/* Get dimensions and extents of the local vectors */
ierr = DAGetInfo(MySim->SimDA, 0, &mx, &my, &mz, 0, 0, 0, 0, 0, 0, 0);CHKERRQ(ierr);
ierr = DAGetCorners(MySim->SimDA, &xs, &ys, &zs, &xm, &ym, &zm);CHKERRQ(ierr);
/* Grab the local coordiantes */
ierr = DAVecGetArray(MySim->SimDA, MySim->x1, &Localx1);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->x2, &Localx2);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->x3, &Localx3);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local geometry */
ierr = DAVecGetArray(MySim->SimDA, MySim->h1, &Localh1);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->h2, &Localh2);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->h3, &Localh3);CHKERRQ(ierr);CHKMEMQ;
/* Grab the Saturations */
ierr = DAVecGetArray(MySim->SimDA, MySim->So, &LocalSo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Sw, &LocalSw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Sg, &LocalSg);CHKERRQ(ierr);CHKMEMQ;
/* Grab the Pressures */
ierr = DAVecGetArray(MySim->SimDA, MySim->Po, &LocalPo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Pw, &LocalPw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Pg, &LocalPg);CHKERRQ(ierr);CHKMEMQ;
/* Grab the Densities */
ierr = DAVecGetArray(MySim->SimDA, MySim->Rhoo, &LocalRhoo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Rhow, &LocalRhow);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Rhog, &LocalRhog);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local viscosities*/
ierr = DAVecGetArray(MySim->SimDA, MySim->Muo, &LocalMuo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Muw, &LocalMuw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Mug, &LocalMug);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local permeabilities */
ierr = DAVecGetArray(MySim->SimDA, MySim->K11, &LocalK11);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->K22, &LocalK22);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->K33, &LocalK33);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local permeabilities */
ierr = DAVecGetArray(MySim->SimDA, MySim->Kro, &LocalKro);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Krw, &LocalKrw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Krg, &LocalKrg);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local capillary pressure */
ierr = DAVecGetArray(MySim->SimDA, MySim->Pcow, &LocalPcow);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Pcog, &LocalPcog);CHKERRQ(ierr);CHKMEMQ;
/* Grab the local data for volume factors at the cell centers */
ierr = DAVecGetArray(MySim->SimDA, MySim->Bo, &LocalBo);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Bw, &LocalBw);CHKERRQ(ierr);CHKMEMQ;
ierr = DAVecGetArray(MySim->SimDA, MySim->Bg, &LocalBg);CHKERRQ(ierr);CHKMEMQ;
/* Get the current rank */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);CHKMEMQ;
/* Now we sync all the flow rates and BHP's across all perforations */
/* At the end of this loop, all information across all wells is uniform and up-to-date */
for (WellID = 0; WellID < MySim->NumberOfWells; WellID++) {
for (PerfIDMine = 0; PerfIDMine
< (MySim->Wells[WellID]).NumberOfPerforations; PerfIDMine++) {
MyI = (MySim->Wells[WellID]).Perforations[PerfIDMine].I;CHKMEMQ;
MyJ = (MySim->Wells[WellID]).Perforations[PerfIDMine].J;CHKMEMQ;
MyK = (MySim->Wells[WellID]).Perforations[PerfIDMine].K;CHKMEMQ;
if (MyI >= xs && MyI < xs+xm && MyJ >= ys && MyJ < ys+ym && MyK >= zs && MyK < zs+zm)
{
MySim->Wells[WellID].Perforations[PerfIDMine].So = LocalSo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Sw = LocalSw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Sg = LocalSg[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Po = LocalPo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Pw = LocalPw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Pg = LocalPg[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Kro = LocalKro[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Krw = LocalKrw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Krg = LocalKrg[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Muo = LocalMuo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Muw = LocalMuw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Mug = LocalMug[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhoo = LocalRhoo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhow = LocalRhow[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhog = LocalRhog[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Pcow = LocalPcow[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Pcog = LocalPcog[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Bo = LocalBo[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Bw = LocalBw[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].Bg = LocalBg[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].x1 = Localx1[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].x2 = Localx2[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].x3 = Localx3[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].h1 = Localh1[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].h2 = Localh2[MyK][MyJ][MyI];CHKMEMQ;
MySim->Wells[WellID].Perforations[PerfIDMine].h3 = Localh3[MyK][MyJ][MyI];CHKMEMQ;
/* we also compute the well indices */
if ((MySim->Wells[WellID]).Perforations[PerfIDMine].Orientation
== PERF_ORIENTATION_X1X1) {
re = 0.14 / 0.5 * sqrt((sqrt(LocalK33[MyK][MyJ][MyI]
/ LocalK22[MyK][MyJ][MyI]) * Localh2[MyK][MyJ][MyI]
* Localh2[MyK][MyJ][MyI] + sqrt(LocalK22[MyK][MyJ][MyI]
/ LocalK33[MyK][MyJ][MyI]) * Localh3[MyK][MyJ][MyI]
* Localh3[MyK][MyJ][MyI])) / (pow(LocalK33[MyK][MyJ][MyI]
/ LocalK22[MyK][MyJ][MyI], 0.25) + pow(LocalK22[MyK][MyJ][MyI]
/ LocalK33[MyK][MyJ][MyI], 0.25));
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex = 2.0 * PI
* Localh1[MyK][MyJ][MyI] * sqrt(LocalK11[MyK][MyJ][MyI]
* LocalK33[MyK][MyJ][MyI]) / (log(re
/ MySim->Wells[WellID].Perforations[PerfIDMine].Rw)
+ MySim->Wells[WellID].Perforations[PerfIDMine].S);
} else if ((MySim->Wells[WellID]).Perforations[PerfIDMine].Orientation
== PERF_ORIENTATION_X2X2) {
re = 0.14 / 0.5 * sqrt((sqrt(LocalK33[MyK][MyJ][MyI]
/ LocalK11[MyK][MyJ][MyI]) * Localh1[MyK][MyJ][MyI]
* Localh1[MyK][MyJ][MyI] + sqrt(LocalK11[MyK][MyJ][MyI]
/ LocalK33[MyK][MyJ][MyI]) * Localh3[MyK][MyJ][MyI]
* Localh3[MyK][MyJ][MyI])) / (pow(LocalK33[MyK][MyJ][MyI]
/ LocalK11[MyK][MyJ][MyI], 0.25) + pow(LocalK11[MyK][MyJ][MyI]
/ LocalK33[MyK][MyJ][MyI], 0.25));
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex = 2.0 * PI
* Localh2[MyK][MyJ][MyI] * sqrt(LocalK11[MyK][MyJ][MyI]
* LocalK33[MyK][MyJ][MyI]) / (log(re
/ MySim->Wells[WellID].Perforations[PerfIDMine].Rw)
+ MySim->Wells[WellID].Perforations[PerfIDMine].S);
} else if ((MySim->Wells[WellID]).Perforations[PerfIDMine].Orientation
== PERF_ORIENTATION_X3X3) {
re = 0.14 / 0.5 * sqrt((sqrt(LocalK22[MyK][MyJ][MyI]
/ LocalK11[MyK][MyJ][MyI]) * Localh1[MyK][MyJ][MyI]
* Localh1[MyK][MyJ][MyI] + sqrt(LocalK11[MyK][MyJ][MyI]
/ LocalK22[MyK][MyJ][MyI]) * Localh2[MyK][MyJ][MyI]
* Localh2[MyK][MyJ][MyI])) / (pow(LocalK22[MyK][MyJ][MyI]
/ LocalK11[MyK][MyJ][MyI], 0.25) + pow(LocalK11[MyK][MyJ][MyI]
/ LocalK22[MyK][MyJ][MyI], 0.25));
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex = 2.0 * PI
* Localh3[MyK][MyJ][MyI] * sqrt(LocalK11[MyK][MyJ][MyI]
* LocalK22[MyK][MyJ][MyI]) / (log(re
/ MySim->Wells[WellID].Perforations[PerfIDMine].Rw)
+ MySim->Wells[WellID].Perforations[PerfIDMine].S);
}
}
i = 0;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Po;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Pw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Pg;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].So;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Sw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Sg;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Rhoo;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Rhow;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Rhog;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Bo;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Bw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Bg;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Kro;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Krw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Krg;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Muo;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Muw;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Mug;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].x1;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].x2;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].x3;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].h1;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].h2;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].h3;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Pcow;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].Pcog;i++;
InBuffer[i] = MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex;
/* Sync properties at the perforation locations */
ierr = MPI_Bcast(InBuffer, 27, MPI_DOUBLE, MySim->Wells[WellID].Perforations[PerfIDMine].OwnerRank, PETSC_COMM_WORLD);CHKERRQ(ierr);CHKMEMQ;
/* and now unpack the data */
i = 0;
MySim->Wells[WellID].Perforations[PerfIDMine].Po = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Pw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Pg = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].So = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Sw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Sg = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhoo = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhow = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Rhog = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Bo = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Bw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Bg = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Kro = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Krw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Krg = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Muo = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Muw = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Mug = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].x1 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].x2 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].x3 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].h1 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].h2 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].h3 = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Pcow = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].Pcog = InBuffer[i];i++;
MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex = InBuffer[i];
PetscPrintf(PETSC_COMM_WORLD, "\n At well: %d Value of WellIndex is:%g", WellID,MySim->Wells[WellID].Perforations[PerfIDMine].WellIndex);
}
}
PetscFunctionReturn(0);
}
PetscErrorCode testCreate3D( )
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 229, d2 = 229, d3 = 229;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,d3,
PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
1,1,
0,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal ***v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym,
ze = info.zs+info.zm;
for (int k = info.zs; k < ze; ++k) {
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[k][j][i] = 1.*i + d1*j + d1*d2*k;
}
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Sequential values filled in petsc vec.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
int lo[3],ld, p = 10;
int patch[10][10][10];
double val;
for (int k = 0; k < d3; k+=p) {
for (int j = 0; j < d2; j+=p) {
for (int i = 0; i < d1; i+=p) {
lo[0] = k;
lo[1] = j;
lo[2] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j + d1*d2*k - val) > .1 )
// printf(".");
printf("(%3.0f,%3.0f) ", 1.*i + d1*j + d1*d2*k, val);
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended NGA_Get() test.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
if( rank == 0 )
{
for (int k = 0; k < d3; ++k) {
printf(">%d\n",k);
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = k; lo[1] = j; lo[2] = i;
val = 1.*i + d1*j + d1*d2*k;
val *= -1;
NGA_Put(ga,lo,lo,&val,&ld);
}
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended NGA_Put() negative seq values.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
for (int k = info.zs; k < ze; ++k) {
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
val = -1 * (1.*i + d1*j + d1*d2*k);
if( PetscAbs( val - v[k][j][i] ) > .1 )
printf(".");
}
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended petsc vec update test.\n"); CHKERRQ(ierr);
if( rank == 0 )
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
