PetscErrorCode FormFunction(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DA da2 = (DA)dmmg->dm;
DALocalInfo info2;
Field2 **x2,**f2;
Vec X2;
PetscFunctionBegin;
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DAGetLocalVector(da2,&X2);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da2,X,INSERT_VALUES,X2);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da2,X,INSERT_VALUES,X2);CHKERRQ(ierr);
/* Access the array inside of X1 */
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted so directly access the memory locations in F */
ierr = DAVecGetArray(da2,F,(void**)&f2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormFunctionLocal2(&info2,0,x2,f2,(void**)user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,F,(void**)&f2);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DARestoreLocalVector(da2,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormInitialGuessComp -
Forms the initial guess for the composite model
Unwraps the global solution vector and passes its local pieces into the user functions
*/
PetscErrorCode FormInitialGuessComp(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DMComposite dm = (DMComposite)dmmg->dm;
Vec X1,X2;
Field1 **x1;
Field2 **x2;
DALocalInfo info1,info2;
DA da1,da2;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2);CHKERRQ(ierr);
/* Access the subvectors in X */
ierr = DMCompositeGetAccess(dm,X,&X1,&X2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal1(&info1,x1);CHKERRQ(ierr);
ierr = FormInitialGuessLocal2(&info2,x2,user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,X,&X1,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Compute the values of the surface mass balance, ice hardness, and sliding coeff. */
static PetscErrorCode FillDistributedParams(AppCtx *user)
{
PetscErrorCode ierr;
PetscInt i,Mx=user->Mx;
PetscScalar x, dum1, dum2, dum3, dum4, dum5, *M, *Bstag, *beta;
PetscFunctionBegin;
/* Compute regular grid exact soln and staggered-grid thickness over the
locally-owned part of the grid */
ierr = DAVecGetArray(user->scalarda,user->M,&M);CHKERRQ(ierr);
ierr = DAVecGetArray(user->scalarda,user->Bstag,&Bstag);CHKERRQ(ierr);
ierr = DAVecGetArray(user->scalarda,user->beta,&beta);CHKERRQ(ierr);
for (i = user->xs; i < user->xs + user->xm; i++) {
x = user->dx * (PetscReal)i; /* = x_i = distance from dome; regular grid point */
ierr = exactN(x, &dum1, &dum2, &dum3, &(M[i]), &dum4, &(beta[i])); CHKERRQ(ierr);
x = x + (user->dx/2.0); /* = x_{i+1/2}; staggered grid point */
if (i < Mx-1) {
ierr = exactN(x, &dum1, &dum2, &dum3, &dum4, &(Bstag[i]), &dum5); CHKERRQ(ierr);
} else {
Bstag[i] = -9999.9999; /* never accessed, and we don't have the value anyway */
}
}
ierr = DAVecRestoreArray(user->scalarda,user->M,&M);CHKERRQ(ierr);
ierr = DAVecRestoreArray(user->scalarda,user->Bstag,&Bstag);CHKERRQ(ierr);
ierr = DAVecRestoreArray(user->scalarda,user->beta,&beta);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void CopyBoundary(State *BHD, Vec gfrom, Vec gto, real zpad)
{
PData PD;
DA da;
int x[3], n[3], i[3], ic[3], dim, k, N[3], border;
PetscScalar ***gfrom_vec, ***gto_vec;
PD = BHD->PD;
da = BHD->da;
FOR_DIM
N[dim] = PD->N[dim];
VecSet(gto, 0.0);
border = (int) ceil( ((PD->interval[1]-PD->interval[0])-(2.*zpad))/PD->h[0]/2. );
/* Get local portion of the grid */
DAGetCorners(da, &(x[0]), &(x[1]), &(x[2]), &(n[0]), &(n[1]), &(n[2]));
DAVecGetArray(da, gfrom, &gfrom_vec);
DAVecGetArray(da, gto, >o_vec);
/* FOR_DIM */
/* { */
/* ic[0]=dim; */
/* ic[1]=(dim+1)%3; */
/* ic[2]=(dim+2)%3; */
/* i[ic[0]] = 0; */
/* if( x[ic[0]] == 0 ) */
/* for( i[ic[1]]=x[ic[1]]; i[ic[1]]<x[ic[1]]+n[ic[1]]; i[ic[1]]++) */
/* for( i[ic[2]]=x[ic[2]]; i[ic[2]]<x[ic[2]]+n[ic[2]]; i[ic[2]]++) */
/* gto_vec[i[2]][i[1]][i[0]] = gfrom_vec[i[2]][i[1]][i[0]]; */
/* i[ic[0]] = N[ic[0]]-1; */
/* if( x[ic[0]]+n[ic[0]] == N[ic[0]] ) */
/* for( i[ic[1]]=x[ic[1]]; i[ic[1]]<x[ic[1]]+n[ic[1]]; i[ic[1]]++) */
/* for( i[ic[2]]=x[ic[2]]; i[ic[2]]<x[ic[2]]+n[ic[2]]; i[ic[2]]++) */
/* gto_vec[i[2]][i[1]][i[0]] = gfrom_vec[i[2]][i[1]][i[0]]; */
/* } */
/* loop over local portion of grid */
for(i[2]=x[2]; i[2]<x[2]+n[2]; i[2]++)
for(i[1]=x[1]; i[1]<x[1]+n[1]; i[1]++)
for(i[0]=x[0]; i[0]<x[0]+n[0]; i[0]++)
{
if( ( i[0]<=border+1 || i[0]>=N[0]-1-border) ||
( i[1]<=border+1 || i[1]>=N[1]-1-border) ||
( i[2]<=border+1 || i[2]>=N[2]-1-border) )
gto_vec[i[2]][i[1]][i[0]] = gfrom_vec[i[2]][i[1]][i[0]];
}
DAVecRestoreArray(da, gfrom, &gfrom_vec);
DAVecRestoreArray(da, gto, >o_vec);
}
PetscErrorCode equation(SNES snes, Vec soln_vec, Vec fun_vec, void* ptr)
{
//data structure will be contained in ptr
Data* data = (Data*) ptr;
Params P = data->P;
DA da = data->da;
DALocalInfo info;
PetscFunctionBegin;
DAGetLocalInfo(da, &info);
Vec sv_local; //need local vector containing ghost point information
DACreateLocalVector(da, &sv_local);
DAGlobalToLocalBegin(da, soln_vec, INSERT_VALUES, sv_local);
DAGlobalToLocalEnd(da, soln_vec, INSERT_VALUES, sv_local);
//to evaluate fuction, the sv_local soln_vec need to be converted into Field arrays
Field **X;
Field **F;
DAVecGetArray(da, sv_local, &X);
DAVecGetArray(da, fun_vec, &F);
//since sv_local is local vector, X will have access to ghost points
PetscInt i, j, Ny = P.Ny;
PetscScalar hx = P.hx; hy = P.hy; x, y;
for(i=info.xs; i<info.xs + info.xm; i++){
for(j=info.ys; j<info.ys+info.ym; j++){
x = i *hx;
y = j *hy;
if(j==0) {
F[j][i].u = X[j][i].u - 0.5*(sin(2*PETSC_PI*x) + 1);
F[j][i].v = X[j][i].v ;
}
else if (j== Ny-1){
F[j][i].u = X[j][i].u;
F[j][i].v = X[j][i].v - 0.5*(sin(2*PETSC_PI*x) + 1);
}
else {
F[j][i].u = lap(u) + X[j][i].u * X[j][i].v ;
F[j][i].v = lap(v) - X[j][i].u * X[j][i].v ;
}
}
}
DAVecRestoreArray(da, fun_vec, &F);
DAVecRestoreArray(da, sv_local, &X);
VecDestory(sv_local);
PetscFuntionReturn(0);
}
/*
FormInitialGuess - Unwraps the global solution vector and passes its local pieces into the user function
*/
PetscErrorCode FormInitialGuess(DMMG dmmg,Vec X)
{
DMComposite dm = (DMComposite)dmmg->dm;
DALocalInfo info1,info2,info3;
DA da1,da2,da3;
FluidField *x1;
PetscScalar **x2;
FuelField **x3;
Vec X1,X2,X3;
PetscErrorCode ierr;
AppCtx *app = (AppCtx*)dmmg->user;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2,&da3);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da3,&info3);
CHKERRQ(ierr);
/* Access the three subvectors in X */
ierr = DMCompositeGetAccess(dm,X,&X1,&X2,&X3);
CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);
CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);
CHKERRQ(ierr);
ierr = DAVecGetArray(da3,X3,(void**)&x3);
CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocalFluid(app,&info1,x1);
CHKERRQ(ierr);
ierr = FormInitialGuessLocalThermal(app,&info2,x2);
CHKERRQ(ierr);
ierr = FormInitialGuessLocalFuel(app,&info3,x3);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da3,X3,(void**)&x3);
CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,X,&X1,&X2,&X3);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ComputeFunction(SNES snes,Vec x,Vec f,void *ctx) {
PetscInt i,Mx,xs,xm; PetscScalar *xx,*ff,hx; DA da = (DA) ctx; Vec xlocal;
DAGetInfo(da,PETSC_IGNORE,&Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
hx = 1.0/(PetscReal)(Mx-1);
DAGetLocalVector(da,&xlocal);DAGlobalToLocalBegin(da,x,INSERT_VALUES,xlocal); DAGlobalToLocalEnd(da,x,INSERT_VALUES,xlocal);
DAVecGetArray(da,xlocal,&xx); DAVecGetArray(da,f,&ff);
DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);
for (i=xs; i<xs+xm; i++) {
if (i == 0 || i == Mx-1) ff[i] = xx[i]/hx;
else ff[i] = (2.0*xx[i] - xx[i-1] - xx[i+1])/hx - hx*PetscExpScalar(xx[i]);
}
DAVecRestoreArray(da,xlocal,&xx); DARestoreLocalVector(da,&xlocal);DAVecRestoreArray(da,f,&ff);
return 0;}
/*
PostCheck - Optional user-defined routine that checks the validity of
candidate steps of a line search method. Set by SNESLineSearchSetPostCheck().
Input Parameters:
snes - the SNES context
ctx - optional user-defined context for private data for the
monitor routine, as set by SNESLineSearchSetPostCheck()
xcurrent - current solution
y - search direction and length
x - the new candidate iterate
Output Parameters:
y - proposed step (search direction and length) (possibly changed)
x - current iterate (possibly modified)
*/
PetscErrorCode PostCheck(SNES snes,Vec xcurrent,Vec y,Vec x,void *ctx,PetscTruth *changed_y,PetscTruth *changed_x)
{
PetscErrorCode ierr;
PetscInt i,iter,xs,xm;
StepCheckCtx *check = (StepCheckCtx*) ctx;
ApplicationCtx *user = check->user;
PetscScalar *xa,*xa_last,tmp;
PetscReal rdiff;
DA da;
PetscFunctionBegin;
*changed_x = PETSC_FALSE;
*changed_y = PETSC_FALSE;
ierr = SNESGetIterationNumber(snes,&iter);CHKERRQ(ierr);
/* iteration 1 indicates we are working on the second iteration */
if (iter > 0) {
da = user->da;
ierr = PetscPrintf(PETSC_COMM_WORLD,"Checking candidate step at iteration %D with tolerance %G\n",iter,check->tolerance);CHKERRQ(ierr);
/* Access local array data */
ierr = DAVecGetArray(da,check->last_step,&xa_last);CHKERRQ(ierr);
ierr = DAVecGetArray(da,x,&xa);CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
/*
If we fail the user-defined check for validity of the candidate iterate,
then modify the iterate as we like. (Note that the particular modification
below is intended simply to demonstrate how to manipulate this data, not
as a meaningful or appropriate choice.)
*/
for (i=xs; i<xs+xm; i++) {
if (!PetscAbsScalar(xa[i])) rdiff = 2*check->tolerance; else rdiff = PetscAbsScalar((xa[i] - xa_last[i])/xa[i]);
if (rdiff > check->tolerance) {
tmp = xa[i];
xa[i] = .5*(xa[i] + xa_last[i]);
*changed_x = PETSC_TRUE;
ierr = PetscPrintf(PETSC_COMM_WORLD," Altering entry %D: x=%G, x_last=%G, diff=%G, x_new=%G\n",
i,PetscAbsScalar(tmp),PetscAbsScalar(xa_last[i]),rdiff,PetscAbsScalar(xa[i]));CHKERRQ(ierr);
}
}
ierr = DAVecRestoreArray(da,check->last_step,&xa_last);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da,x,&xa);CHKERRQ(ierr);
}
ierr = VecCopy(x,check->last_step);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormFunctionComp - Unwraps the input vector and passes its local ghosted pieces into the user function
*/
PetscErrorCode FormFunctionComp(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DMComposite dm = (DMComposite)dmmg->dm;
DALocalInfo info1,info2;
DA da1,da2;
Field1 **x1,**f1;
Field2 **x2,**f2;
Vec X1,X2,F1,F2;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DMCompositeGetLocalVectors(dm,&X1,&X2);CHKERRQ(ierr);
ierr = DMCompositeScatter(dm,X,X1,X2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted and directly access the memory locations in F */
ierr = DMCompositeGetAccess(dm,F,&F1,&F2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of F */
ierr = DAVecGetArray(da1,F1,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,F2,(void**)&f2);CHKERRQ(ierr);
/* Evaluate local user provided function */
user->x2 = x2; /* used by FormFunctionLocal1() */
ierr = FormFunctionLocal1(&info1,x1,f1,(void**)user);CHKERRQ(ierr);
user->x1 = x1; /* used by FormFunctionLocal2() */
ierr = FormFunctionLocal2(&info2,x2,f2,(void**)user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,F1,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,F2,(void**)&f2);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,F,&F1,&F2);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DMCompositeRestoreLocalVectors(dm,&X1,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FluidFieldDivergence_3D( DALocalInfo info,
Coor d, Vec US, Vec VS, Vec WS, Vec DIV )
{
int i,j,k;
PetscReal xe, xs, ye, ys, ze, zs;
PetscReal ***u, ***v, ***w, ***div;
PetscErrorCode ierr;
PetscFunctionBegin;
xs = info.xs;
xs = xs == 0 ? 2 : xs;
xe = info.xs+info.xm;
xe = xe == info.mx ? info.mx-3 : xe;
ys = info.ys;
ys = ys == 0 ? 2 : ys;
ye = info.ys+info.ym;
ye = ye == info.my ? info.my-3 : ye;
zs = info.zs;
zs = zs == 0 ? 2 : zs;
ze = info.zs+info.zm;
ze = ze == info.mz ? info.mz-3 : ze;
ierr = DAVecGetArray(info.da,US,&u); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,VS,&v); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,WS,&w); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,DIV,&div); CHKERRQ(ierr);
for( k = zs; k < ze; ++k)
{
for (j = ys; j < ye; ++j)
{
for (i = xs; i < xe; ++i)
{
div[k][j][i] = (u[k][j][i+1] - u[k][j][i]) / d.x +
(v[k][j+1][i] - v[k][j][i]) / d.y +
(w[k+1][j][i] - w[k][j][i]) / d.z;
}
}
}
ierr = DAVecRestoreArray(info.da,US,&u); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,VS,&v); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,WS,&w); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,DIV,&div); CHKERRQ(ierr);
PetscLogFlops(8*(xe-xs)*(ye-ys)*(ze-zs));
PetscFunctionReturn(0);
}
PetscErrorCode testCreate2D()
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 1453, d2 = 1451;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal **v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym;
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[j][i] = 1.*i + d1 * j;
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Updated local portion with DAVec\n");
PetscBarrier(0);
{
double *da_ptr;
VecGetArray(vec, &da_ptr);
double *ptr;
int low[2],hi[2],ld;
NGA_Distribution(ga,GA_Nodeid(),low,hi);
NGA_Access(ga,low,hi,&ptr,&ld);
printf("[%d] ga:%p\tda:%p\tdiff:%p\n", GA_Nodeid(), ptr, da_ptr, (ptr-da_ptr) );
NGA_Release_update(ga,low,hi);
}
int lo[2],ld;
double val;
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = j;
lo[1] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j - val) > .1 )
printf(".");
// printf("[%d] (%3.0f,%3.0f)\n", GA_Nodeid(), 1.*i + d1*j, val);
}
}
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
/*
MSA_InitialPoint - Calculates the initial guess in one of three ways.
Input Parameters:
. user - user-defined application context
. X - vector for initial guess
Output Parameters:
. X - newly computed initial guess
*/
static int MSA_InitialPoint(AppCtx * user, Vec X)
{
int info;
PetscInt start2=-1,i,j;
PetscReal start1=0;
PetscTruth flg1,flg2;
info = PetscOptionsGetReal(PETSC_NULL,"-start",&start1,&flg1); CHKERRQ(info);
info = PetscOptionsGetInt(PETSC_NULL,"-random",&start2,&flg2); CHKERRQ(info);
if (flg1){ /* The zero vector is reasonable */
info = VecSet(X, start1); CHKERRQ(info);
} else if (flg2 && start2>0){ /* Try a random start between -0.5 and 0.5 */
PetscRandom rctx; PetscScalar np5=-0.5;
info = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);
CHKERRQ(info);
for (i=0; i<start2; i++){
info = VecSetRandom(X, rctx); CHKERRQ(info);
}
info = PetscRandomDestroy(rctx); CHKERRQ(info);
info = VecShift(X, np5); CHKERRQ(info);
} else { /* Take an average of the boundary conditions */
PetscInt xs,xm,ys,ym;
PetscInt mx=user->mx,my=user->my;
PetscScalar **x;
/* Get local mesh boundaries */
info = DAGetCorners(user->da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL); CHKERRQ(info);
/* Get pointers to vector data */
info = DAVecGetArray(user->da,X,(void**)&x);
/* Perform local computations */
for (j=ys; j<ys+ym; j++){
for (i=xs; i< xs+xm; i++){
x[j][i] = ( ((j+1)*user->bottom[i-xs+1]+(my-j+1)*user->top[i-xs+1])/(my+2)+
((i+1)*user->left[j-ys+1]+(mx-i+1)*user->right[j-ys+1])/(mx+2))/2.0;
}
}
/* Restore vectors */
info = DAVecRestoreArray(user->da,X,(void**)&x); CHKERRQ(info);
info = PetscLogFlops(9*xm*ym); CHKERRQ(info);
}
return 0;
}
PetscErrorCode FormFunction1(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DA da1 = (DA)dmmg->dm;
DALocalInfo info1;
Field1 **x1,**f1;
Vec X1;
Field2 **x2;
PetscFunctionBegin;
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DAGetLocalVector(da1,&X1);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da1,X,INSERT_VALUES,X1);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da1,X,INSERT_VALUES,X1);CHKERRQ(ierr);
/* Access the arrays inside X1 */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted so directly access the memory locations in F */
ierr = DAVecGetArray(da1,F,(void**)&f1);CHKERRQ(ierr);
/* Evaluate local user provided function */
if (user->nsolve){
x2 = user->x2;
ierr = FormFunctionLocal1(&info1,x1,x2,f1,(void**)user);CHKERRQ(ierr);
} else {
ierr = FormFunctionLocal1(&info1,x1,0,f1,(void**)user);CHKERRQ(ierr);
}
ierr = DAVecRestoreArray(da1,F,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DARestoreLocalVector(da1,&X1);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start %s\n", __FILE__); CHKERRQ(ierr);
MPI_Comm comm = PETSC_COMM_WORLD;
DA da;
Mat mat;
int MX = 100;
int dof = 3;
ierr = PetscPrintf(comm,"Started Assembly\n"); CHKERRQ(ierr);
ierr = MatMake(MX,dof,&da, &mat); CHKERRQ(ierr);
ierr = MatWrite("J",mat); CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Finished Assembly\n"); CHKERRQ(ierr);
Vec rhs, sol;
ierr = DACreateGlobalVector(da,&rhs); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&sol); CHKERRQ(ierr);
int i,j;
int xs,ys,xm,ym;
Field **array;
ierr = DAVecGetArray(da,rhs,&array); CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL); CHKERRQ(ierr);
for (j = xs; j < xm; ++j) {
for (i = ys; i < ym; ++i) {
if( 3*MX/4 > i && i > MX/4 &&
3*MX/4 > j && j > MX/4 )
{
array[j][i].u = 100.;
array[j][i].v = 100.;
}
}
}
ierr = DAVecRestoreArray(da,rhs,&array); CHKERRQ(ierr);
KSP ksp;
ierr = KSPCreate(comm,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY); CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,mat,mat,DIFFERENT_NONZERO_PATTERN); CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
//Split pressure from velocity
PC pc;
ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr);
ierr = PCFieldSplitSetFields(pc,2,(int[]){0,1}); CHKERRQ(ierr);
/* This function releases the memory allocated for the immersed structure */
void Immersed_destroy(Immersed *q_immersed, MAC_grid *grid) {
int ierr;
free(q_immersed->ib_nodes);
/* release the pointer to the Vector */
ierr = DAVecRestoreArray(grid->DA_3D, q_immersed->L_global_index, (void ***)&q_immersed->global_index); PETScErrAct(ierr);
ierr = VecDestroy(q_immersed->G_global_index); PETScErrAct(ierr);
ierr = VecDestroy(q_immersed->L_global_index); PETScErrAct(ierr);
free(q_immersed);
}
/* Put a not-unreasonable initial guess in Hu. */
static PetscErrorCode FillInitial(AppCtx *user, Vec *vHu)
{
PetscErrorCode ierr;
PetscInt i;
Node *Hu;
PetscFunctionBegin;
ierr = DAVecGetArray(user->da,*vHu,&Hu);CHKERRQ(ierr);
for (i = user->xs; i < user->xs + user->xm; i++) {
Hu[i].H = 1000.0;
Hu[i].u = 100.0 / user->secpera;
}
ierr = DAVecRestoreArray(user->da,*vHu,&Hu);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int FormBratuFunction(SNES snes, Vec v, Vec f, void* ctx)
{
UserBratuCtx* bratu = (UserBratuCtx *) ctx;
DA da = bratu->da;
double lambda = bratu->lambda;
double h = bratu->h;
Vec lv;
int i,j;
int lli, llj, ni, nj;
const double **varr;
double **fvarr;
DAGetCorners(da, &lli, &llj, 0, &ni, &nj, 0);
DAGetLocalVector(da, &lv);
DAGlobalToLocalBegin(da, v, INSERT_VALUES, lv);
DAGlobalToLocalEnd(da, v, INSERT_VALUES,lv);
DAVecGetArray(da, lv, (void**) &varr);
DAVecGetArray(da, f, (void**) &fvarr);
for(j=llj; j<llj+nj; j++){
for(i=lli; i<lli+ni; i++){
if(i==0 || j==0 ||i=bratu->n+1 || j==bratu->n+1){
fvarr[j][i] = 0.0;
}
else{
fvarr[j][i] = -(varr[j-1][i] + varr[j][i-1] + varr[j+1][i] + varr[j][i+1] - 4*varr[j][i])/(h*h) - lambda*exp(varr[j][i]);
}
}
}
DAVecRestoreArray(da, f, (void**) &fvarr);
DAVecRestoreArray(da, lv, (void**) &varr);
DARestoreLocalVector(da, &lv);
return 0;
}
PetscErrorCode FluidFieldDivergence_2D( DALocalInfo info, Coor d,
Vec US, Vec VS, Vec DIV )
{
int i,j;
PetscReal xe, xs, ye, ys;
PetscReal **u, **v, **div;
PetscErrorCode ierr;
PetscFunctionBegin;
xs = info.xs;
xs = xs == 0 ? 2 : xs;
xe = info.xs+info.xm;
xe = xe == info.mx ? info.mx-3 : xe;
ys = info.ys;
ys = ys == 0 ? 2 : ys;
ye = info.ys+info.ym;
ye = ye == info.my ? info.my-3 : ye;
ierr = DAVecGetArray(info.da,US,&u); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,VS,&v); CHKERRQ(ierr);
ierr = DAVecGetArray(info.da,DIV,&div); CHKERRQ(ierr);
for (j = ys; j < ye; ++j)
{
for (i = xs; i < xe; ++i)
{
div[j][i] = (u[j][i+1] - u[j][i]) / d.x +
(v[j+1][i] - v[j][i]) / d.y;
}
}
ierr = DAVecRestoreArray(info.da,US,&u); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,VS,&v); CHKERRQ(ierr);
ierr = DAVecRestoreArray(info.da,DIV,&div); CHKERRQ(ierr);
PetscLogFlops(5*(xe-xs)*(ye-ys));
PetscFunctionReturn(0);
}
/*
FormInitialGuessLocal - Forms initial approximation for this process
Input Parameters:
user - user-defined application context
X - vector (DA local vector)
Output Parameter:
X - vector with the local values set
*/
PetscErrorCode FormInitialGuessLocal(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
PetscReal grashof,dx;
Field **x;
grashof = user->grashof;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
dx = 1.0/(mx-1);
/*
Get local grid boundaries (for 2-dimensional DA):
xs, ys - starting grid indices (no ghost points)
xm, ym - widths of local grid (no ghost points)
*/
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
/*
Get a pointer to vector data.
- For default PETSc vectors, VecGetArray() returns a pointer to
the data array. Otherwise, the routine is implementation dependent.
- You MUST call VecResetArraystoreArray() when you no longer need access to
the array.
*/
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
/*
Compute initial guess over the locally owned part of the grid
Initial condition is motionless fluid and equilibrium temperature
U = V = Omega = 0.0; Temp[x_i, y_j] = x_i;
*/
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].u = 0.0;
x[j][i].v = 0.0;
x[j][i].omega = 0.0;
x[j][i].temp = (grashof>0)*i*dx;
}
}
/* Restore vector */
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
int main(int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start\n"); CHKERRQ(ierr);
int d1 = 5;
ierr = PetscOptionsSetValue("-da_grid_x","5"); CHKERRQ(ierr);
int d2 = 7;
ierr = PetscOptionsSetValue("-da_grid_y","7"); CHKERRQ(ierr);
FluidField f;
ierr = FluidFieldCreate(&f); CHKERRQ(ierr);
iCoor s = {d1,d2,0};
Grid g;
ierr = GridCreate(s,&g); CHKERRQ(ierr);
PetscReal *p;
VecGetArray(f->p, &p);
for (int i = 0; i < g->n.x*g->n.y; ++i)
{
g->v1[i] = i;
p[i] = i;
}
PetscReal **pp;
DAVecGetArray(f->da, f->p, &pp);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
printf( "%1.0f, %1.0f\t", g->v2[j][i], pp[j][i]);
}
printf("\n");
}
DAVecRestoreArray(f->da, f->p, &pp);
ierr = GridDestroy(g); CHKERRQ(ierr);
ierr = FluidFieldDestroy(f); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "End\n"); CHKERRQ(ierr);
ierr = PetscFinalize(); CHKERRQ(ierr);
return 0;
}
/* Compute the exact thickness and velocity on the regular grid. */
static PetscErrorCode FillExactSoln(AppCtx *user)
{
PetscErrorCode ierr;
PetscInt i;
PetscScalar x, dum1, dum2, dum3, dum4;
Node *Hu;
PetscFunctionBegin;
/* Compute regular grid exact soln and staggered-grid thickness over the
locally-owned part of the grid */
ierr = DAVecGetArray(user->da,user->Huexact,&Hu);CHKERRQ(ierr);
for (i = user->xs; i < user->xs + user->xm; i++) {
x = user->dx * (PetscReal)i; /* = x_i = distance from dome */
ierr = exactN(x, &(Hu[i].H), &dum1, &(Hu[i].u), &dum2, &dum3, &dum4); CHKERRQ(ierr);
}
ierr = DAVecRestoreArray(user->da,user->Huexact,&Hu);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int Initialize(DMMG *dmmg)
/* ------------------------------------------------------------------- */
{
AppCtx *user = (AppCtx*)dmmg[0]->user;
Parameter *param;
DA da;
PetscReal PI = 3.14159265358979323846;
PetscReal sigma,xc,zc;
PetscReal dx=user->grid->dx,dz=user->grid->dz;
int i,j,ierr,is,js,im,jm;
Field **x;
ierr = PetscBagGetData(user->bag,(void**)¶m);
CHKERRQ(ierr);
sigma=param->sigma;
xc=param->xctr;
zc=param->zctr;
/* Get the DA and grid */
da = (DA)(dmmg[0]->dm);
ierr = DAGetCorners(da,&is,&js,PETSC_NULL,&im,&jm,PETSC_NULL);
CHKERRQ(ierr);
ierr = DAVecGetArray(da,user->Xold,(void**)&x);
CHKERRQ(ierr);
for (j=js; j<js+jm; j++) {
for (i=is; i<is+im; i++) {
if (param->flow_type == SHEAR_CELL) {
x[j][i].u = -sin(PI*i*dx)*cos(PI*j*dz)/dx;
x[j][i].w = sin(PI*j*dz)*cos(PI*i*dx)/dz;
} else {
x[j][i].u = 0.0;
x[j][i].w = -1.0/dz;
}
x[j][i].phi = 100*exp(-0.5*((i*dx-xc)*(i*dx-xc)+(j*dz-zc)*(j*dz-zc))/sigma/sigma);
}
}
/* restore the grid to it's vector */
ierr = DAVecRestoreArray(da,user->Xold,(void**)&x);
CHKERRQ(ierr);
ierr = VecCopy(user->Xold, DMMGGetx(dmmg));
CHKERRQ(ierr);
return 0;
}
PetscErrorCode ComputeJacobian(SNES snes,Vec x,Mat *J,Mat *B,MatStructure *flag,void *ctx){
DA da = (DA) ctx; PetscInt i,Mx,xm,xs; PetscScalar hx,*xx; Vec xlocal;
DAGetInfo(da,PETSC_IGNORE,&Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
hx = 1.0/(PetscReal)(Mx-1);
DAGetLocalVector(da,&xlocal);DAGlobalToLocalBegin(da,x,INSERT_VALUES,xlocal); DAGlobalToLocalEnd(da,x,INSERT_VALUES,xlocal);
DAVecGetArray(da,xlocal,&xx);
DAGetCorners(da,&xs,PETSC_NULL,PETSC_NULL,&xm,PETSC_NULL,PETSC_NULL);
for (i=xs; i<xs+xm; i++) {
if (i == 0 || i == Mx-1) { MatSetValue(*J,i,i,1.0/hx,INSERT_VALUES);}
else {
MatSetValue(*J,i,i-1,-1.0/hx,INSERT_VALUES);
MatSetValue(*J,i,i,2.0/hx - hx*PetscExpScalar(xx[i]),INSERT_VALUES);
MatSetValue(*J,i,i+1,-1.0/hx,INSERT_VALUES);
}
}
MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY); MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY); *flag = SAME_NONZERO_PATTERN;
DAVecRestoreArray(da,xlocal,&xx);DARestoreLocalVector(da,&xlocal);
return 0;}
PetscErrorCode FormInitialGuess(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DA da2 = (DA)dmmg->dm;
Field2 **x2;
DALocalInfo info2;
PetscFunctionBegin;
/* Access the arrays inside of X */
ierr = DAVecGetArray(da2,X,(void**)&x2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal2(&info2,x2,user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X,(void**)&x2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode doit(DA da,Vec global)
{
PetscErrorCode ierr;
PetscInt i,j,k,M,N,dof;
ierr = DAGetInfo(da,0,&M,&N,0,0,0,0,&dof,0,0,0);CHKERRQ(ierr);
{
struct {PetscScalar inside[dof];} **mystruct;
ierr = DAVecGetArray(da,global,(void*) &mystruct);
for ( i=0; i<N; i++) {
for ( j=0; j<M; j++) {
for ( k=0; k<dof; k++) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"%d %d %G\n",i,j,mystruct[i][j].inside[0]);CHKERRQ(ierr);
mystruct[i][j].inside[1] = 2.1;
}
}
}
ierr = DAVecRestoreArray(da,global,(void*) &mystruct);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/* Form initial guess for Physic 1 */
PetscErrorCode FormInitialGuessLocal1(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
Field1 **x;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].u = 0.0;
x[j][i].v = 0.0;
x[j][i].omega = 0.0;
}
}
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
PetscErrorCode FormInitialGuess1(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DA da1 = (DA)dmmg->dm;
Field1 **x1;
DALocalInfo info1;
PetscFunctionBegin;
if (user->nsolve) PetscFunctionReturn(0);
printf(" FormInitialGuess1 ... user.nsolve %d\n",user->nsolve);
/* Access the array inside of X */
ierr = DAVecGetArray(da1,X,(void**)&x1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal1(&info1,x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X,(void**)&x1);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* Form initial guess for Physic 2 */
PetscErrorCode FormInitialGuessLocal2(DMMG dmmg,Vec X)
{
AppCtx *user = (AppCtx*)dmmg->user;
DA da = (DA)dmmg->dm;
PetscInt i,j,mx,xs,ys,xm,ym;
PetscErrorCode ierr;
PetscReal grashof,dx;
Field2 **x;
grashof = user->grashof;
ierr = DAGetInfo(da,0,&mx,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
dx = 1.0/(mx-1);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL);CHKERRQ(ierr);
ierr = DAVecGetArray(da,X,&x);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
x[j][i].temp = (grashof>0)*i*dx;
}
}
ierr = DAVecRestoreArray(da,X,&x);CHKERRQ(ierr);
return 0;
}
PetscErrorCode testCreate3D( )
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 229, d2 = 229, d3 = 229;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,d3,
PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
1,1,
0,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal ***v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym,
ze = info.zs+info.zm;
for (int k = info.zs; k < ze; ++k) {
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[k][j][i] = 1.*i + d1*j + d1*d2*k;
}
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Sequential values filled in petsc vec.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
int lo[3],ld, p = 10;
int patch[10][10][10];
double val;
for (int k = 0; k < d3; k+=p) {
for (int j = 0; j < d2; j+=p) {
for (int i = 0; i < d1; i+=p) {
lo[0] = k;
lo[1] = j;
lo[2] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j + d1*d2*k - val) > .1 )
// printf(".");
printf("(%3.0f,%3.0f) ", 1.*i + d1*j + d1*d2*k, val);
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended NGA_Get() test.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
if( rank == 0 )
{
for (int k = 0; k < d3; ++k) {
printf(">%d\n",k);
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = k; lo[1] = j; lo[2] = i;
val = 1.*i + d1*j + d1*d2*k;
val *= -1;
NGA_Put(ga,lo,lo,&val,&ld);
}
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended NGA_Put() negative seq values.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
for (int k = info.zs; k < ze; ++k) {
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
val = -1 * (1.*i + d1*j + d1*d2*k);
if( PetscAbs( val - v[k][j][i] ) > .1 )
printf(".");
}
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended petsc vec update test.\n"); CHKERRQ(ierr);
if( rank == 0 )
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
PetscErrorCode vizGA2DA()
{
PetscErrorCode ierr;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
int d1 = 40, d2 = 50;
DA da;
Vec vec;
const PetscInt *lx, *ly, *lz;
PetscInt m,n,p;
DALocalInfo info;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr);
ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr);
/**/
ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
for (int i = 0; i < m; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]);
}
for (int i = 0; i < n; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]);
}
/**/
int ga = GA_Create_handle();
int ndim = 2;
int dims[2] = {d2,d1};
GA_Set_data(ga,2,dims,MT_DBL);
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < n; i++ )
{
map[i] = ly[i-1] + map[i-1];
}
map[n] = 0;
for( int i = n+1; i < m+n; i++ )
{
map[i] = lx[i-n-1] + map[i-1];
}
/* correct ordering, but nodeid's dont line up with mpi rank for petsc's da
* DA: +---+---+ GA: +---+---+
* +-2-+-3-+ +-1-+-3-+
* +---+---+ +---+---+
* +-0-+-1-+ +-0-+-2-+
* +---+---+ +---+---+
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < m; i++ )
{
map[i] = lx[i] + map[i-1];
}
map[m] = 0;
for( int i = m+1; i < m+n; i++ )
{
map[i] = ly[i-m] + map[i-1];
}
*/
int block[2] = {n,m};
GA_Set_irreg_distr(ga,map,block);
ierr = GA_Allocate( ga );
if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr);
if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr);
if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1);
GA_Print_distribution(ga);
int lo[2], hi[2];
NGA_Distribution(ga,rank,lo,hi);
if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 ||
lo[0] != info.ys || hi[0] != info.ys+info.ym-1 )
{
PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n",
rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1);
}
PetscBarrier(0);
PetscSynchronizedFlush(PETSC_COMM_WORLD);
AO ao;
DAGetAO(da,&ao);
if( rank == 0 )
{
int *idx, len = d1*d2;
PetscReal *val;
PetscMalloc(sizeof(PetscReal)*len, &val);
PetscMalloc(sizeof(int)*len, &idx);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
idx[i + d1*j] = i + d1*j;
val[i + d1*j] = i + d1*j;
}
}
AOApplicationToPetsc(ao,len,idx);
VecSetValues(vec,len,idx,val,INSERT_VALUES);
int a[2], b[2],ld[1]={0};
double c = 0;
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
a[0] = j;
a[1] = i;
// printf("%5.0f ",c);
NGA_Put(ga,a,a,&c,ld);
c++;
}
}
}
// GA_Print(ga);
VecAssemblyBegin(vec);
VecAssemblyEnd(vec);
int ld;
double *ptr;
NGA_Access(ga,lo,hi,&ptr,&ld);
PetscReal **d;
int c=0;
ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr);
for (int j = info.ys; j < info.ys+info.ym; ++j)
{
for (int i = info.xs; i < info.xs+info.xm; ++i)
{
if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] )
GA_Error("DA array is not equal to GA array",1);
// printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]);
c++;
}
}
ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr);
c=0;
PetscReal *v;
int start, end;
VecGetOwnershipRange(vec, &start, &end);
VecGetArray( vec, &v );
for( int i = start; i < end; i++)
{
// printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") );
}
VecRestoreArray( vec, &v );
NGA_Release_update(ga,lo,hi);
Vec gada;
VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada);
VecView(gada,PETSC_VIEWER_STDOUT_SELF);
GA_Destroy(ga);
ierr = VecDestroy(vec); CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
