/*
FormInitialGuessComp -
Forms the initial guess for the composite model
Unwraps the global solution vector and passes its local pieces into the user functions
*/
PetscErrorCode FormInitialGuessComp(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DMComposite dm = (DMComposite)dmmg->dm;
Vec X1,X2;
Field1 **x1;
Field2 **x2;
DALocalInfo info1,info2;
DA da1,da2;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2);CHKERRQ(ierr);
/* Access the subvectors in X */
ierr = DMCompositeGetAccess(dm,X,&X1,&X2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal1(&info1,x1);CHKERRQ(ierr);
ierr = FormInitialGuessLocal2(&info2,x2,user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,X,&X1,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormInitialGuess - Unwraps the global solution vector and passes its local pieces into the user function
*/
PetscErrorCode FormInitialGuess(DMMG dmmg,Vec X)
{
DMComposite dm = (DMComposite)dmmg->dm;
DALocalInfo info1,info2,info3;
DA da1,da2,da3;
FluidField *x1;
PetscScalar **x2;
FuelField **x3;
Vec X1,X2,X3;
PetscErrorCode ierr;
AppCtx *app = (AppCtx*)dmmg->user;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2,&da3);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da3,&info3);
CHKERRQ(ierr);
/* Access the three subvectors in X */
ierr = DMCompositeGetAccess(dm,X,&X1,&X2,&X3);
CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);
CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);
CHKERRQ(ierr);
ierr = DAVecGetArray(da3,X3,(void**)&x3);
CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocalFluid(app,&info1,x1);
CHKERRQ(ierr);
ierr = FormInitialGuessLocalThermal(app,&info2,x2);
CHKERRQ(ierr);
ierr = FormInitialGuessLocalFuel(app,&info3,x3);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da3,X3,(void**)&x3);
CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,X,&X1,&X2,&X3);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode SetPressureBC( FluidField f )
{
int i,j;
PetscReal **mask, m;
PetscReal ***rhs,p;
DALocalInfo g;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscInfo(0,"Entering SetPressureBC()"); CHKERRQ(ierr);
ierr = DAGetLocalInfo(f->daB, &g); CHKERRQ(ierr);
ierr = GridGet(f->mask, &mask); CHKERRQ(ierr);
ierr = DAVecGetArrayDOF(f->daV,f->rhs,&rhs); CHKERRQ(ierr);
for (j = g.ys; j < g.ys+g.ym; ++j) {
for (i = g.xs; i < g.xs+g.xm; ++i) {
m = mask[j][i];
if( m > -1.5 ) continue;
if( PetscAbs(m+2) < .01 ) p = 20;
if( PetscAbs(m+3) < .01 ) p = 19;
if( PetscAbs(m+4) < .01 ) p = 10;
rhs[j][i][CELL_CENTER] = p;
}
}
ierr = DAVecRestoreArrayDOF(f->daV,f->rhs,&rhs); CHKERRQ(ierr);
ierr = PetscInfo(0,"Exiting SetPressureBC()"); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FluidFieldDivergence(FluidField f)
{
DALocalInfo info;
Vec u,v,w; // Local, ghosted vectors
PetscErrorCode ierr;
PetscFunctionBegin;
PetscLogEventBegin(EVENT_FluidFieldDivergence,0,0,0,0);
// PetscLogEventRegister(&EVENT_FluidFieldDivergence,"FluidFieldDivergence", 0);
ierr = DAGetLocalInfo(f->da,&info); CHKERRQ(ierr);
ierr = DAGetLocalVector(f->da,&u); CHKERRQ(ierr);
ierr = DAGetLocalVector(f->da,&v); CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(f->da,f->u,INSERT_VALUES,u); CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd( f->da,f->u,INSERT_VALUES,u); CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(f->da,f->v,INSERT_VALUES,v); CHKERRQ(ierr);
//TODO: interleaving work with communication here a possible source of optimization?
ierr = DAGlobalToLocalEnd( f->da,f->v,INSERT_VALUES,v); CHKERRQ(ierr);
if( f->is3D )
{
ierr = DAGetLocalVector(f->da,&w); CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(f->da,f->w,INSERT_VALUES,w); CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd( f->da,f->w,INSERT_VALUES,w); CHKERRQ(ierr);
FluidFieldDivergence_3D(info, f->d, u, v, w, f->div);
ierr = DARestoreLocalVector(f->da,&w); CHKERRQ(ierr);
} else {
FluidFieldDivergence_2D(info, f->d, u, v, f->div);
}
ierr = DARestoreLocalVector(f->da,&u); CHKERRQ(ierr);
ierr = DARestoreLocalVector(f->da,&v); CHKERRQ(ierr);
PetscLogEventEnd(EVENT_FluidFieldDivergence,0,0,0,0);
PetscFunctionReturn(0);
}
PetscErrorCode FormFunction(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DA da2 = (DA)dmmg->dm;
DALocalInfo info2;
Field2 **x2,**f2;
Vec X2;
PetscFunctionBegin;
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DAGetLocalVector(da2,&X2);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da2,X,INSERT_VALUES,X2);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da2,X,INSERT_VALUES,X2);CHKERRQ(ierr);
/* Access the array inside of X1 */
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted so directly access the memory locations in F */
ierr = DAVecGetArray(da2,F,(void**)&f2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormFunctionLocal2(&info2,0,x2,f2,(void**)user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,F,(void**)&f2);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DARestoreLocalVector(da2,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormFunctionComp - Unwraps the input vector and passes its local ghosted pieces into the user function
*/
PetscErrorCode FormFunctionComp(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DMComposite dm = (DMComposite)dmmg->dm;
DALocalInfo info1,info2;
DA da1,da2;
Field1 **x1,**f1;
Field2 **x2,**f2;
Vec X1,X2,F1,F2;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DMCompositeGetLocalVectors(dm,&X1,&X2);CHKERRQ(ierr);
ierr = DMCompositeScatter(dm,X,X1,X2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted and directly access the memory locations in F */
ierr = DMCompositeGetAccess(dm,F,&F1,&F2);CHKERRQ(ierr);
/* Access the arrays inside the subvectors of F */
ierr = DAVecGetArray(da1,F1,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecGetArray(da2,F2,(void**)&f2);CHKERRQ(ierr);
/* Evaluate local user provided function */
user->x2 = x2; /* used by FormFunctionLocal1() */
ierr = FormFunctionLocal1(&info1,x1,f1,(void**)user);CHKERRQ(ierr);
user->x1 = x1; /* used by FormFunctionLocal2() */
ierr = FormFunctionLocal2(&info2,x2,f2,(void**)user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,F1,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,F2,(void**)&f2);CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,F,&F1,&F2);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);CHKERRQ(ierr);
ierr = DMCompositeRestoreLocalVectors(dm,&X1,&X2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode testCreate2D()
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 1453, d2 = 1451;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal **v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym;
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[j][i] = 1.*i + d1 * j;
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Updated local portion with DAVec\n");
PetscBarrier(0);
{
double *da_ptr;
VecGetArray(vec, &da_ptr);
double *ptr;
int low[2],hi[2],ld;
NGA_Distribution(ga,GA_Nodeid(),low,hi);
NGA_Access(ga,low,hi,&ptr,&ld);
printf("[%d] ga:%p\tda:%p\tdiff:%p\n", GA_Nodeid(), ptr, da_ptr, (ptr-da_ptr) );
NGA_Release_update(ga,low,hi);
}
int lo[2],ld;
double val;
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = j;
lo[1] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j - val) > .1 )
printf(".");
// printf("[%d] (%3.0f,%3.0f)\n", GA_Nodeid(), 1.*i + d1*j, val);
}
}
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
PetscErrorCode equation(SNES snes, Vec soln_vec, Vec fun_vec, void* ptr)
{
//data structure will be contained in ptr
Data* data = (Data*) ptr;
Params P = data->P;
DA da = data->da;
DALocalInfo info;
PetscFunctionBegin;
DAGetLocalInfo(da, &info);
Vec sv_local; //need local vector containing ghost point information
DACreateLocalVector(da, &sv_local);
DAGlobalToLocalBegin(da, soln_vec, INSERT_VALUES, sv_local);
DAGlobalToLocalEnd(da, soln_vec, INSERT_VALUES, sv_local);
//to evaluate fuction, the sv_local soln_vec need to be converted into Field arrays
Field **X;
Field **F;
DAVecGetArray(da, sv_local, &X);
DAVecGetArray(da, fun_vec, &F);
//since sv_local is local vector, X will have access to ghost points
PetscInt i, j, Ny = P.Ny;
PetscScalar hx = P.hx; hy = P.hy; x, y;
for(i=info.xs; i<info.xs + info.xm; i++){
for(j=info.ys; j<info.ys+info.ym; j++){
x = i *hx;
y = j *hy;
if(j==0) {
F[j][i].u = X[j][i].u - 0.5*(sin(2*PETSC_PI*x) + 1);
F[j][i].v = X[j][i].v ;
}
else if (j== Ny-1){
F[j][i].u = X[j][i].u;
F[j][i].v = X[j][i].v - 0.5*(sin(2*PETSC_PI*x) + 1);
}
else {
F[j][i].u = lap(u) + X[j][i].u * X[j][i].v ;
F[j][i].v = lap(v) - X[j][i].u * X[j][i].v ;
}
}
}
DAVecRestoreArray(da, fun_vec, &F);
DAVecRestoreArray(da, sv_local, &X);
VecDestory(sv_local);
PetscFuntionReturn(0);
}
PetscErrorCode FormInitialGuess(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DA da2 = (DA)dmmg->dm;
Field2 **x2;
DALocalInfo info2;
PetscFunctionBegin;
/* Access the arrays inside of X */
ierr = DAVecGetArray(da2,X,(void**)&x2);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal2(&info2,x2,user);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X,(void**)&x2);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FormFunction1(SNES snes,Vec X,Vec F,void *ctx)
{
PetscErrorCode ierr;
DMMG dmmg = (DMMG)ctx;
AppCtx *user = (AppCtx*)dmmg->user;
DA da1 = (DA)dmmg->dm;
DALocalInfo info1;
Field1 **x1,**f1;
Vec X1;
Field2 **x2;
PetscFunctionBegin;
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X */
ierr = DAGetLocalVector(da1,&X1);CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da1,X,INSERT_VALUES,X1);CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da1,X,INSERT_VALUES,X1);CHKERRQ(ierr);
/* Access the arrays inside X1 */
ierr = DAVecGetArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
/* Access the subvectors in F.
These are not ghosted so directly access the memory locations in F */
ierr = DAVecGetArray(da1,F,(void**)&f1);CHKERRQ(ierr);
/* Evaluate local user provided function */
if (user->nsolve){
x2 = user->x2;
ierr = FormFunctionLocal1(&info1,x1,x2,f1,(void**)user);CHKERRQ(ierr);
} else {
ierr = FormFunctionLocal1(&info1,x1,0,f1,(void**)user);CHKERRQ(ierr);
}
ierr = DAVecRestoreArray(da1,F,(void**)&f1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);CHKERRQ(ierr);
ierr = DARestoreLocalVector(da1,&X1);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FormInitialGuess1(DMMG dmmg,Vec X)
{
PetscErrorCode ierr;
AppCtx *user = (AppCtx*)dmmg->user;
DA da1 = (DA)dmmg->dm;
Field1 **x1;
DALocalInfo info1;
PetscFunctionBegin;
if (user->nsolve) PetscFunctionReturn(0);
printf(" FormInitialGuess1 ... user.nsolve %d\n",user->nsolve);
/* Access the array inside of X */
ierr = DAVecGetArray(da1,X,(void**)&x1);CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormInitialGuessLocal1(&info1,x1);CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X,(void**)&x1);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode testCreate3D( )
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 229, d2 = 229, d3 = 229;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,d3,
PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
1,1,
0,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal ***v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym,
ze = info.zs+info.zm;
for (int k = info.zs; k < ze; ++k) {
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[k][j][i] = 1.*i + d1*j + d1*d2*k;
}
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Sequential values filled in petsc vec.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
int lo[3],ld, p = 10;
int patch[10][10][10];
double val;
for (int k = 0; k < d3; k+=p) {
for (int j = 0; j < d2; j+=p) {
for (int i = 0; i < d1; i+=p) {
lo[0] = k;
lo[1] = j;
lo[2] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j + d1*d2*k - val) > .1 )
// printf(".");
printf("(%3.0f,%3.0f) ", 1.*i + d1*j + d1*d2*k, val);
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended NGA_Get() test.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
if( rank == 0 )
{
for (int k = 0; k < d3; ++k) {
printf(">%d\n",k);
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = k; lo[1] = j; lo[2] = i;
val = 1.*i + d1*j + d1*d2*k;
val *= -1;
NGA_Put(ga,lo,lo,&val,&ld);
}
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended NGA_Put() negative seq values.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
for (int k = info.zs; k < ze; ++k) {
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
val = -1 * (1.*i + d1*j + d1*d2*k);
if( PetscAbs( val - v[k][j][i] ) > .1 )
printf(".");
}
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended petsc vec update test.\n"); CHKERRQ(ierr);
if( rank == 0 )
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
PetscErrorCode vizGA2DA()
{
PetscErrorCode ierr;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
int d1 = 40, d2 = 50;
DA da;
Vec vec;
const PetscInt *lx, *ly, *lz;
PetscInt m,n,p;
DALocalInfo info;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr);
ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr);
/**/
ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
for (int i = 0; i < m; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]);
}
for (int i = 0; i < n; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]);
}
/**/
int ga = GA_Create_handle();
int ndim = 2;
int dims[2] = {d2,d1};
GA_Set_data(ga,2,dims,MT_DBL);
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < n; i++ )
{
map[i] = ly[i-1] + map[i-1];
}
map[n] = 0;
for( int i = n+1; i < m+n; i++ )
{
map[i] = lx[i-n-1] + map[i-1];
}
/* correct ordering, but nodeid's dont line up with mpi rank for petsc's da
* DA: +---+---+ GA: +---+---+
* +-2-+-3-+ +-1-+-3-+
* +---+---+ +---+---+
* +-0-+-1-+ +-0-+-2-+
* +---+---+ +---+---+
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < m; i++ )
{
map[i] = lx[i] + map[i-1];
}
map[m] = 0;
for( int i = m+1; i < m+n; i++ )
{
map[i] = ly[i-m] + map[i-1];
}
*/
int block[2] = {n,m};
GA_Set_irreg_distr(ga,map,block);
ierr = GA_Allocate( ga );
if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr);
if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr);
if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1);
GA_Print_distribution(ga);
int lo[2], hi[2];
NGA_Distribution(ga,rank,lo,hi);
if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 ||
lo[0] != info.ys || hi[0] != info.ys+info.ym-1 )
{
PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n",
rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1);
}
PetscBarrier(0);
PetscSynchronizedFlush(PETSC_COMM_WORLD);
AO ao;
DAGetAO(da,&ao);
if( rank == 0 )
{
int *idx, len = d1*d2;
PetscReal *val;
PetscMalloc(sizeof(PetscReal)*len, &val);
PetscMalloc(sizeof(int)*len, &idx);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
idx[i + d1*j] = i + d1*j;
val[i + d1*j] = i + d1*j;
}
}
AOApplicationToPetsc(ao,len,idx);
VecSetValues(vec,len,idx,val,INSERT_VALUES);
int a[2], b[2],ld[1]={0};
double c = 0;
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
a[0] = j;
a[1] = i;
// printf("%5.0f ",c);
NGA_Put(ga,a,a,&c,ld);
c++;
}
}
}
// GA_Print(ga);
VecAssemblyBegin(vec);
VecAssemblyEnd(vec);
int ld;
double *ptr;
NGA_Access(ga,lo,hi,&ptr,&ld);
PetscReal **d;
int c=0;
ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr);
for (int j = info.ys; j < info.ys+info.ym; ++j)
{
for (int i = info.xs; i < info.xs+info.xm; ++i)
{
if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] )
GA_Error("DA array is not equal to GA array",1);
// printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]);
c++;
}
}
ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr);
c=0;
PetscReal *v;
int start, end;
VecGetOwnershipRange(vec, &start, &end);
VecGetArray( vec, &v );
for( int i = start; i < end; i++)
{
// printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") );
}
VecRestoreArray( vec, &v );
NGA_Release_update(ga,lo,hi);
Vec gada;
VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada);
VecView(gada,PETSC_VIEWER_STDOUT_SELF);
GA_Destroy(ga);
ierr = VecDestroy(vec); CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void Compute_Distance_Function(UserCtx *user)
{
DALocalInfo info;
PetscInt i, j, k;
DAGetLocalInfo(user->da, &info);
PetscInt mx = info.mx, my = info.my, mz = info.mz;
PetscInt xs = info.xs, xe = xs + info.xm;
PetscInt ys = info.ys, ye = ys + info.ym;
PetscInt zs = info.zs, ze = zs + info.zm;
PetscInt lxs = xs, lxe = xe;
PetscInt lys = ys, lye = ye;
PetscInt lzs = zs, lze = ze;
PetscReal ***level;
if (xs==0) lxs = xs+1;
if (ys==0) lys = ys+1;
if (zs==0) lzs = zs+1;
if (xe==mx) lxe = xe-1;
if (ye==my) lye = ye-1;
if (ze==mz) lze = ze-1;
int iter=0;
Init_LevelSet_Vectors(user);
Distance_Function_IC(user);
Vec D;
double norm=100, norm_old;
dtau = dx_min*0.25;
//dtau = dx_min * 0.5;
VecDuplicate(LevelSet, &D);
do {
norm_old = norm;
PetscPrintf(PETSC_COMM_WORLD, "\nSolving Distance %d... (dtau=%f)\n", iter, dtau);
//Solve_Distance_Explicit(user);
Solve_Distance(user, iter);
VecWAXPY(D, -1., LevelSet_o, LevelSet);
VecNorm(D, NORM_INFINITY, &norm);
PetscPrintf(PETSC_COMM_WORLD, "\nNorm=%.5e\n\n", norm);
PetscReal ***level;
DAVecGetArray(user->da, LevelSet, &level);
for (k=zs; k<ze; k++)
for (j=ys; j<ye; j++)
for (i=xs; i<xe; i++) if(level[k][j][i]<0) level[k][j][i]=0.01;
DAVecRestoreArray(user->da, LevelSet, &level);
if(iter > 10 && (fabs(norm)<1.e-5 || iter>1000) ) break;
//if(iter>10 && iter%50==0) dtau *=1.04;
VecCopy(LevelSet, LevelSet_o);
iter++;
} while(1);
VecDestroy(D);
Vec lLevelset;
PetscReal ***llevel;
VecDuplicate(user->lP, &lLevelset);
DAGlobalToLocalBegin(user->da, LevelSet, INSERT_VALUES, lLevelset);
DAGlobalToLocalEnd(user->da, LevelSet, INSERT_VALUES, lLevelset);
DAVecGetArray(user->da, LevelSet, &level);
DAVecGetArray(user->da, lLevelset, &llevel);
for (k=zs; k<ze; k++)
for (j=ys; j<ye; j++)
for (i=xs; i<xe; i++) {
int c=k, b=j, a=i, flag=0;
// Neumann conditions
if(i==0) a=1, flag=1;
if(i==mx-1) a=mx-2, flag=1;
if(j==0) b=1, flag=1;
if(j==my-1) b=my-2, flag=1;
if(k==0) c=1, flag=1;
if(k==mz-1) c=mz-2, flag=1;
if(i_periodic && i==0) a=mx-2, flag=1;
else if(i_periodic && i==mx-1) a=1, flag=1;
if(j_periodic && j==0) b=my-2, flag=1;
else if(j_periodic && j==my-1) b=1, flag=1;
if(k_periodic && k==0) c=mz-2, flag=1;
else if(k_periodic && k==mz-1) c=1, flag=1;
if(ii_periodic && i==0) a=-2, flag=1;
else if(ii_periodic && i==mx-1) a=mx+1, flag=1;
if(jj_periodic && j==0) b=-2, flag=1;
else if(jj_periodic && j==my-1) b=my+1, flag=1;
if(kk_periodic && k==0) c=-2, flag=1;
else if(kk_periodic && k==mz-1) c=mz+1, flag=1;
if(flag) level[k][j][i] = llevel[c][b][a];
}
DAVecRestoreArray(user->da, LevelSet, &level);
DAVecRestoreArray(user->da, lLevelset, &llevel);
VecDestroy(lLevelset);
VecCopy(LevelSet, user->Distance);
Destroy_LevelSet_Vectors(user);
};
void Solve_Distance_Explicit(UserCtx *user)
{
DALocalInfo info;
PetscInt xs, xe, ys, ye, zs, ze; // Local grid information
PetscInt mx, my, mz; // Dimensions in three directions
PetscInt i, j, k;
PetscInt lxs, lxe, lys, lye, lzs, lze;
PetscReal ***level;
DAGetLocalInfo(user->da, &info);
mx = info.mx; my = info.my; mz = info.mz;
xs = info.xs; xe = xs + info.xm;
ys = info.ys; ye = ys + info.ym;
zs = info.zs; ze = zs + info.zm;
lxs = xs; lxe = xe;
lys = ys; lye = ye;
lzs = zs; lze = ze;
if (xs==0) lxs = xs+1;
if (ys==0) lys = ys+1;
if (zs==0) lzs = zs+1;
if (xe==mx) lxe = xe-1;
if (ye==my) lye = ye-1;
if (ze==mz) lze = ze-1;
DAGlobalToLocalBegin(user->da, LevelSet, INSERT_VALUES, user->lLevelset);
DAGlobalToLocalEnd(user->da, LevelSet, INSERT_VALUES, user->lLevelset);
DAVecGetArray(user->da, user->lLevelset, &level);
if(xs==0 || xe==mx) {
int from, to;
for (k=lzs; k<lze; k++)
for (j=lys; j<lye; j++) {
if(xs==0) {
i = 1, from = i, to = 0;
if(i_periodic) from = mx-2;
else if(ii_periodic) from = -2;
level[k][j][to] = level[k][j][from];
}
if(xe==mx) {
i = mx-2, from = i, to = mx-1;
if(i_periodic) from = 1;
else if(ii_periodic) from = mx+1;
level[k][j][to] = level[k][j][from];
}
}
}
if(ys==0 || ye==my) {
int from, to;
for (k=lzs; k<lze; k++)
for (i=lxs; i<lxe; i++) {
if(ys==0) {
j = 1, from = j, to = 0;
if(j_periodic) from = my-2;
else if(jj_periodic) from = -2;
level[k][to][i] = level[k][from][i];
}
if(ye==my) {
j = my-2, from = j, to = my-1;
if(j_periodic) from = 1;
else if(jj_periodic) from = my+1;
level[k][to][i] = level[k][from][i];
}
}
}
if(zs==0 || ze==mz) {
int from, to;
for (j=lys; j<lye; j++)
for (i=lxs; i<lxe; i++) {
if(zs==0) {
k = 1, from = k, to = 0;
if(k_periodic) from = mz-2;
else if(kk_periodic) from = -2;
level[to][j][i] = level[from][j][i];
}
if(ze==mz) {
k = mz-2, from = k, to = mz-1;
if(k_periodic) from = 1;
else if(kk_periodic) from = mz+1;
level[to][j][i] = level[from][j][i];
}
}
}
DAVecRestoreArray(user->da, user->lLevelset, &level);
Vec Rhs;
VecDuplicate(user->P, &Rhs);
Distance_Function_RHS(user, Rhs, 1);
VecAXPY(LevelSet, dtau, Rhs);
VecDestroy(Rhs);
return;
}
void Distance_Function_IC(UserCtx *user)
{
DA da = user->da, fda = user->fda;
DALocalInfo info;
PetscInt xs, xe, ys, ye, zs, ze; // Local grid information
PetscInt mx, my, mz; // Dimensions in three directions
PetscInt i, j, k, ibi;
PetscInt lxs, lxe, lys, lye, lzs, lze;
Cmpnts ***csi, ***eta, ***zet;
PetscReal ***nvert, ***L0, ***aj;
DAGetLocalInfo(da, &info);
mx = info.mx; my = info.my; mz = info.mz;
xs = info.xs; xe = xs + info.xm;
ys = info.ys; ye = ys + info.ym;
zs = info.zs; ze = zs + info.zm;
lxs = xs; lxe = xe;
lys = ys; lye = ye;
lzs = zs; lze = ze;
if (xs==0) lxs = xs+1;
if (ys==0) lys = ys+1;
if (zs==0) lzs = zs+1;
if (xe==mx) lxe = xe-1;
if (ye==my) lye = ye-1;
if (ze==mz) lze = ze-1;
DAVecGetArray(da, user->lNvert, &nvert);
DAVecGetArray(da, LevelSet0, &L0);
DAVecGetArray(fda, user->lCsi, &csi);
DAVecGetArray(fda, user->lEta, &eta);
DAVecGetArray(fda, user->lZet, &zet);
DAVecGetArray(da, user->lAj, &aj);
if(immersed)
for(ibi=0; ibi<NumberOfBodies; ibi++)
{
//IBMNodes *ibm = &ibm_ptr[ibi];
IBMListNode *current;
current = user->ibmlist[ibi].head;
while (current) {
IBMInfo *ibminfo = ¤t->ibm_intp;
current = current->next;
double sb = ibminfo->d_s;
i=ibminfo->ni, j= ibminfo->nj, k = ibminfo->nk;
L0[k][j][i] = sb;
};
}
for (k=zs; k<ze; k++)
for (j=ys; j<ye; j++)
for (i=xs; i<xe; i++) {
int flag=0;
double dist=1.e10;
if(nvert[k][j][i]>0.1) continue;
if((user->bctype[0]==1|| user->bctype[0]==-1 || user->bctype[0]==-2) && i==1){
double area=sqrt(csi[k][j][i].x*csi[k][j][i].x+csi[k][j][i].y*csi[k][j][i].y+csi[k][j][i].z*csi[k][j][i].z);
dist = PetscMin(dist,0.5/aj[k][j][i]/area);
flag=1;
L0[k][j][i-1] = dist;
}
if((user->bctype[1]==1|| user->bctype[1]==-1 || user->bctype[1]==-2) && i==mx-2 ) {
double area=sqrt(csi[k][j][i].x*csi[k][j][i].x+csi[k][j][i].y*csi[k][j][i].y+csi[k][j][i].z*csi[k][j][i].z);
dist = PetscMin(dist,0.5/aj[k][j][i]/area);
flag=1;
L0[k][j][i+1] = dist;
}
if((user->bctype[2]==1|| user->bctype[2]==-1 || user->bctype[2]==-2) && j==1){
double area=sqrt(eta[k][j][i].x*eta[k][j][i].x+eta[k][j][i].y*eta[k][j][i].y+eta[k][j][i].z*eta[k][j][i].z);
dist = PetscMin(dist,0.5/aj[k][j][i]/area);
flag=1;
L0[k][j-1][i] = dist;
}
if((user->bctype[3]==1|| user->bctype[3]==-1 || user->bctype[3]==-2 || user->bctype[3]==12) &&j==my-2){
double area=sqrt(eta[k][j][i].x*eta[k][j][i].x+eta[k][j][i].y*eta[k][j][i].y+eta[k][j][i].z*eta[k][j][i].z);
dist = PetscMin(dist,0.5/aj[k][j][i]/area);
flag=1;
L0[k][j+1][i] = dist;
}
if((user->bctype[4]==1|| user->bctype[4]==-1 || user->bctype[4]==-2) && k==1){
double area=sqrt(zet[k][j][i].x*zet[k][j][i].x+zet[k][j][i].y*zet[k][j][i].y+zet[k][j][i].z*zet[k][j][i].z);
dist = PetscMin(dist,0.5/aj[k][j][i]/area);
flag=1;
L0[k-1][j][i] = dist;
}
if((user->bctype[5]==1 || user->bctype[5]==-1 || user->bctype[5]==-2) && k==mz-2 ) {
double area=sqrt(zet[k][j][i].x*zet[k][j][i].x+zet[k][j][i].y*zet[k][j][i].y+zet[k][j][i].z*zet[k][j][i].z);
dist = PetscMin(dist,0.5/aj[k][j][i]/area);
flag=1;
L0[k+1][j][i] = dist;
}
if(flag) L0[k][j][i] = dist;
else L0[k][j][i] = 0.05;
//if(i==0 || i==mx-1 || j==0 || j==my-1 || k==0 || k==mz-1 ) L0[k][j][i]=0;
}
DAVecRestoreArray(da, user->lNvert, &nvert);
DAVecRestoreArray(da, LevelSet0, &L0);
DAVecRestoreArray(fda, user->lCsi, &csi);
DAVecRestoreArray(fda, user->lEta, &eta);
DAVecRestoreArray(fda, user->lZet, &zet);
DAVecRestoreArray(da, user->lAj, &aj);
//DALocalToGlobal(da, lLevelSet0, INSERT_VALUES, LevelSet); // copy
//VecCopy (LevelSet, LevelSet_o);
VecCopy (LevelSet0, LevelSet);
VecCopy (LevelSet0, LevelSet_o);
};
/*
Here is my custom preconditioner
Capital vectors: X, X1 are global vectors
Small vectors: x, x1 are local ghosted vectors
Prefixed a: ax1, aY1 are arrays that access the vector values (either local (ax1) or global aY1)
*/
PetscErrorCode MyPCApply(PC pc,Vec X,Vec Y)
{
AppCtx *app;
PetscErrorCode ierr;
Vec X1,X2,X3,x1,x2,Y1,Y2,Y3;
DALocalInfo info1,info2,info3;
DA da1,da2,da3;
PetscInt i,j;
FluidField *ax1,*aY1;
PetscScalar **ax2,**aY2;
PetscFunctionBegin;
ierr = PCShellGetContext(pc,(void**)&app);
CHKERRQ(ierr);
/* obtain information about the three meshes */
ierr = DMCompositeGetEntries(app->pack,&da1,&da2,&da3);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da3,&info3);
CHKERRQ(ierr);
/* get ghosted version of fluid and thermal conduction, global for phi and C */
ierr = DMCompositeGetAccess(app->pack,X,&X1,&X2,&X3);
CHKERRQ(ierr);
ierr = DMCompositeGetLocalVectors(app->pack,&x1,&x2,PETSC_NULL);
CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da1,X1,INSERT_VALUES,x1);
CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da1,X1,INSERT_VALUES,x1);
CHKERRQ(ierr);
ierr = DAGlobalToLocalBegin(da2,X2,INSERT_VALUES,x2);
CHKERRQ(ierr);
ierr = DAGlobalToLocalEnd(da2,X2,INSERT_VALUES,x2);
CHKERRQ(ierr);
/* get global version of result vector */
ierr = DMCompositeGetAccess(app->pack,Y,&Y1,&Y2,&Y3);
CHKERRQ(ierr);
/* pull out the phi and C values */
ierr = VecStrideGather(X3,0,app->dx,INSERT_VALUES);
CHKERRQ(ierr);
ierr = VecStrideGather(X3,1,app->c,INSERT_VALUES);
CHKERRQ(ierr);
/* update C via formula 38; put back into return vector */
ierr = VecAXPY(app->c,0.0,app->dx);
CHKERRQ(ierr);
ierr = VecScale(app->c,1.0);
CHKERRQ(ierr);
ierr = VecStrideScatter(app->c,1,Y3,INSERT_VALUES);
CHKERRQ(ierr);
/* form the right hand side of the phi equation; solve system; put back into return vector */
ierr = VecAXPBY(app->dx,0.0,1.0,app->c);
CHKERRQ(ierr);
ierr = DMMGSolve(app->fdmmg);
CHKERRQ(ierr);
ierr = VecStrideScatter(app->dy,0,Y3,INSERT_VALUES);
CHKERRQ(ierr);
/* access the ghosted x1 and x2 as arrays */
ierr = DAVecGetArray(da1,x1,&ax1);
CHKERRQ(ierr);
ierr = DAVecGetArray(da2,x2,&ax2);
CHKERRQ(ierr);
/* access global y1 and y2 as arrays */
ierr = DAVecGetArray(da1,Y1,&aY1);
CHKERRQ(ierr);
ierr = DAVecGetArray(da2,Y2,&aY2);
CHKERRQ(ierr);
for (i=info1.xs; i<info1.xs+info1.xm; i++) {
aY1[i].prss = ax1[i].prss;
aY1[i].ergg = ax1[i].ergg;
aY1[i].ergf = ax1[i].ergf;
aY1[i].alfg = ax1[i].alfg;
aY1[i].velg = ax1[i].velg;
aY1[i].velf = ax1[i].velf;
}
for (j=info2.ys; j<info2.ys+info2.ym; j++) {
for (i=info2.xs; i<info2.xs+info2.xm; i++) {
aY2[j][i] = ax2[j][i];
}
}
ierr = DAVecRestoreArray(da1,x1,&ax1);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,x2,&ax2);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,Y1,&aY1);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,Y2,&aY2);
CHKERRQ(ierr);
ierr = DMCompositeRestoreLocalVectors(app->pack,&x1,&x2,PETSC_NULL);
CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(app->pack,X,&X1,&X2,&X3);
CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(app->pack,Y,&Y1,&Y2,&Y3);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
FormFunction - Unwraps the input vector and passes its local ghosted pieces into the user function
*/
PetscErrorCode FormFunction(SNES snes,Vec X,Vec F,void *ctx)
{
DMMG dmmg = (DMMG)ctx;
DMComposite dm = (DMComposite)dmmg->dm;
DALocalInfo info1,info2,info3;
DA da1,da2,da3;
FluidField *x1,*f1;
PetscScalar **x2,**f2;
FuelField **x3,**f3;
Vec X1,X2,X3,F1,F2,F3;
PetscErrorCode ierr;
PetscInt i,j;
AppCtx *app = (AppCtx*)dmmg->user;
PetscFunctionBegin;
ierr = DMCompositeGetEntries(dm,&da1,&da2,&da3);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da1,&info1);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da2,&info2);
CHKERRQ(ierr);
ierr = DAGetLocalInfo(da3,&info3);
CHKERRQ(ierr);
/* Get local vectors to hold ghosted parts of X; then fill in the ghosted vectors from the unghosted global vector X */
ierr = DMCompositeGetLocalVectors(dm,&X1,&X2,&X3);
CHKERRQ(ierr);
ierr = DMCompositeScatter(dm,X,X1,X2,X3);
CHKERRQ(ierr);
/* Access the arrays inside the subvectors of X */
ierr = DAVecGetArray(da1,X1,(void**)&x1);
CHKERRQ(ierr);
ierr = DAVecGetArray(da2,X2,(void**)&x2);
CHKERRQ(ierr);
ierr = DAVecGetArray(da3,X3,(void**)&x3);
CHKERRQ(ierr);
/*
Ghost points for periodicity are used to "force" inflow/outflow fluid boundary conditions
Note that there is no periodicity; we define periodicity to "cheat" and have ghost spaces to store "exterior to boundary" values
*/
/* FLUID */
if (info1.gxs == -3) { /* 3 points at left end of line */
for (i=-3; i<0; i++) {
x1[i].prss = app->prin;
x1[i].ergg = app->ugin;
x1[i].ergf = app->ufin;
x1[i].alfg = app->agin;
x1[i].velg = app->vgin;
x1[i].velf = app->vfin;
}
}
if (info1.gxs+info1.gxm == info1.mx+3) { /* 3 points at right end of line */
for (i=info1.mx; i<info1.mx+3; i++) {
x1[i].prss = app->prout;
x1[i].ergg = app->ugout;
x1[i].ergf = app->ufout;
x1[i].alfg = app->agout;
x1[i].velg = app->vgi;
x1[i].velf = app->vfi;
}
}
/* Thermal */
if (info2.gxs == -1) { /* left side of domain */
for (j=info2.gys; j<info2.gys+info2.gym; j++) {
x2[j][-1] = app->twi;
}
}
if (info2.gxs+info2.gxm == info2.mx+1) { /* right side of domain */
for (j=info2.gys; j<info2.gys+info2.gym; j++) {
x2[j][info2.mx] = app->twi;
}
}
/* FUEL */
if (info3.gxs == -1) { /* left side of domain */
for (j=info3.gys; j<info3.gys+info3.gym; j++) {
x3[j][-1].phii = app->phii;
x3[j][-1].prei = app->prei;
}
}
if (info3.gxs+info3.gxm == info3.mx+1) { /* right side of domain */
for (j=info3.gys; j<info3.gys+info3.gym; j++) {
x3[j][info3.mx].phii = app->phii;
x3[j][info3.mx].prei = app->prei;
}
}
if (info3.gys == -1) { /* bottom of domain */
for (i=info3.gxs; i<info3.gxs+info3.gxm; i++) {
x3[-1][i].phii = app->phii;
x3[-1][i].prei = app->prei;
}
}
if (info3.gys+info3.gym == info3.my+1) { /* top of domain */
for (i=info3.gxs; i<info3.gxs+info3.gxm; i++) {
x3[info3.my][i].phii = app->phii;
x3[info3.my][i].prei = app->prei;
}
}
/* Access the three subvectors in F: these are not ghosted and directly access the memory locations in F */
ierr = DMCompositeGetAccess(dm,F,&F1,&F2,&F3);
CHKERRQ(ierr);
/* Access the arrays inside the subvectors of F */
ierr = DAVecGetArray(da1,F1,(void**)&f1);
CHKERRQ(ierr);
ierr = DAVecGetArray(da2,F2,(void**)&f2);
CHKERRQ(ierr);
ierr = DAVecGetArray(da3,F3,(void**)&f3);
CHKERRQ(ierr);
/* Evaluate local user provided function */
ierr = FormFunctionLocalFluid(&info1,x1,f1);
CHKERRQ(ierr);
ierr = FormFunctionLocalThermal(&info2,x2,f2);
CHKERRQ(ierr);
ierr = FormFunctionLocalFuel(&info3,x3,f3);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,X1,(void**)&x1);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,X2,(void**)&x2);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da3,X3,(void**)&x3);
CHKERRQ(ierr);
ierr = DMCompositeRestoreLocalVectors(dm,&X1,&X2,&X3);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da1,F1,(void**)&f1);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da2,F2,(void**)&f2);
CHKERRQ(ierr);
ierr = DAVecRestoreArray(da3,F3,(void**)&f3);
CHKERRQ(ierr);
ierr = DMCompositeRestoreAccess(dm,F,&F1,&F2,&F3);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
//int BenchmarkBoundaryChecks( int n, PetscLogDouble t_bulk, PetscLogDouble )
int PetscMain()
{
int i,j,k, n=128;
PetscLogDouble t1,t2,s1,s2;
Vec U,V,W,DIV1,DIV2;
PetscReal ***u,***v,***w,***div1, ***div2;
DA da;
DALocalInfo info;
PetscErrorCode ierr;
ierr = DACreate3d(PETSC_COMM_SELF,//MPI Communicator
DA_NONPERIODIC, //DA_NONPERIODIC, DA_XPERIODIC, DA_YPERIODIC, DA_XYPERIODIC
DA_STENCIL_STAR, //DA_STENCIL_BOX or DA_STENCIL_STAR
n,n,n, //Global array dimension
1,1,1,//Number procs per dim
1, //Number of chemical species
1, //stencil width
0,0,0, //specific array of nodes
&da); CHKERRQ(ierr);
DACreateGlobalVector(da,&U);
DACreateGlobalVector(da,&V);
DACreateGlobalVector(da,&W);
DACreateGlobalVector(da,&DIV1);
DACreateGlobalVector(da,&DIV2);
VecSet(DIV1,0);
VecSet(DIV2,0);
DAVecGetArray(da,U,&u);
DAVecGetArray(da,V,&v);
DAVecGetArray(da,W,&w);
DAVecGetArray(da,DIV1,&div1);
DAVecGetArray(da,DIV2,&div2);
DAGetLocalInfo(da,&info);
PetscBarrier(0);
for( k = 0; k < n; ++k)
{
for( j = 0; j < n; ++j)
{
for( i = 0; i < n; ++i)
{
u[k][j][i] = i * (i-n) * j * (j-n) * k * (k-n);
v[k][j][i] = 1 - u[k][j][i];
w[k][j][i] = u[k][j][i] * v[k][j][i];
}
}
}
PetscBarrier(0);
PetscGetTime(&t1);
for( k = 1; k < n-1; ++k)
{
for( j = 1; j < n-1; ++j)
{
for( i = 1; i < n-1; ++i)
{
div1[k][j][i] = u[k][j][i+1] - u[k][j][i-1] +
v[k][j+1][i] - v[k][j-1][i] +
w[k+1][j][i] - w[k-1][j][i];
div1[k][j][i] /= 2;
}
}
}
PetscGetTime(&t2);
PetscReal uE,uW,vN,vS,wF,wB;
PetscReal hx,hy,hz;
PetscBarrier(0);
PetscGetTime(&s1);
for( k = 1; k < n-1; ++k)
{
for( j = 1; j < n-1; ++j)
{
for( i = 1; i < n-1; ++i)
{
/* uE = i == info.mx-1 ? u[k][j][i] : u[k][j][i+1];
uW = i == 0 ? u[k][j][i] : u[k][j][i-1];
vN = j == info.my-1 ? v[k][j][i] : v[k][j+1][i];
vS = j == 0 ? v[k][j][i] : v[k][j-1][i];
wB = k == info.mz-1 ? w[k][j][i] : w[k+1][j][i];
wF = k == 0 ? w[k][j][i] : w[k-1][j][i];
*/
if( i == info.mx-1 ) { uE = u[k][j][i]; hx= 1; }else{ uE = u[k][j][i+1];hx= 2;}
if( i == info.mx-1 ) { uE = u[k][j][i]; hx= 1; }else{ uE = u[k][j][i+1];hx= 2;}
if( j == info.mx-1 ) { uE = u[k][j][i]; hy= 1; }else{ uE = u[k][j][i+1];hy= 2;}
if( j == info.mx-1 ) { uE = u[k][j][i]; hy= 1; }else{ uE = u[k][j][i+1];hy= 2;}
if( k == info.mx-1 ) { uE = u[k][j][i]; hz= 1; }else{ uE = u[k][j][i+1];hz= 2;}
if( k == info.mx-1 ) { uE = u[k][j][i]; hz= 1; }else{ uE = u[k][j][i+1];hz= 2;}
div2[k][j][i] = uE - uW + vN - vS + wB - wF;
div2[k][j][i] /= 2;
// printf("%f\t%f\t%f\n",div1[k][j][i], div2[k][j][i], div2[k][j][i] - div1[k][j][i]);
}
}
}
PetscGetTime(&s2);
DAVecRestoreArray(da,DIV1,&div1);
DAVecRestoreArray(da,DIV2,&div2);
VecAXPY(DIV1,-1,DIV2);
PetscReal norm;
VecNorm(DIV1,NORM_2,&norm);
printf("BULK: %f\nIF's: %f\nDIFF:\t%f\nRATIO:\t%f\nnorm: %f\n", (t2-t1), (s2-s1), (s2-s1)-(t2-t1),(s2-s1)/(t2-t1),norm);
}
