/*@C
DMDAGetRay - Returns a vector on process zero that contains a row or column of the values in a DMDA vector
Collective on DMDA
Input Parameters:
+ da - the distributed array
. vec - the vector
. dir - Cartesian direction, either DMDA_X, DMDA_Y, or DMDA_Z
- gp - global grid point number in this direction
Output Parameters:
+ newvec - the new vector that can hold the values (size zero on all processes except process 0)
- scatter - the VecScatter that will map from the original vector to the slice
Level: advanced
Notes:
All processors that share the DMDA must call this with the same gp value
.keywords: distributed array, get, processor subset
@*/
PetscErrorCode DMDAGetRay(DM da,DMDADirection dir,PetscInt gp,Vec *newvec,VecScatter *scatter)
{
PetscMPIInt rank;
DM_DA *dd = (DM_DA*)da->data;
PetscErrorCode ierr;
IS is;
AO ao;
Vec vec;
PetscInt *indices,i,j;
PetscFunctionBegin;
if (dd->dim == 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Cannot get slice from 1d DMDA");
if (dd->dim == 3) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Cannot get slice from 3d DMDA");
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)da),&rank);CHKERRQ(ierr);
if (!rank) {
if (dir == DMDA_Y) {
ierr = PetscMalloc(dd->w*dd->M*sizeof(PetscInt),&indices);CHKERRQ(ierr);
indices[0] = gp*dd->M*dd->w;
for (i=1; i<dd->M*dd->w; i++) indices[i] = indices[i-1] + 1;
ierr = AOApplicationToPetsc(ao,dd->M*dd->w,indices);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,newvec);CHKERRQ(ierr);
ierr = VecSetBlockSize(*newvec,dd->w);CHKERRQ(ierr);
ierr = VecSetSizes(*newvec,dd->M*dd->w,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(*newvec,VECSEQ);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,dd->w*dd->M,indices,PETSC_OWN_POINTER,&is);CHKERRQ(ierr);
} else if (dir == DMDA_X) {
ierr = PetscMalloc(dd->w*dd->N*sizeof(PetscInt),&indices);CHKERRQ(ierr);
indices[0] = dd->w*gp;
for (j=1; j<dd->w; j++) indices[j] = indices[j-1] + 1;
for (i=1; i<dd->N; i++) {
indices[i*dd->w] = indices[i*dd->w-1] + dd->w*dd->M - dd->w + 1;
for (j=1; j<dd->w; j++) indices[i*dd->w + j] = indices[i*dd->w + j - 1] + 1;
}
ierr = AOApplicationToPetsc(ao,dd->w*dd->N,indices);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,newvec);CHKERRQ(ierr);
ierr = VecSetBlockSize(*newvec,dd->w);CHKERRQ(ierr);
ierr = VecSetSizes(*newvec,dd->N*dd->w,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(*newvec,VECSEQ);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,dd->w*dd->N,indices,PETSC_OWN_POINTER,&is);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Unknown DMDADirection");
} else {
ierr = VecCreateSeq(PETSC_COMM_SELF,0,newvec);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,0,0,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
}
ierr = DMGetGlobalVector(da,&vec);CHKERRQ(ierr);
ierr = VecScatterCreate(vec,is,*newvec,NULL,scatter);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&vec);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Given a DMDA generates a VecScatter context that will deliver a slice
of the global vector to each processor. In this example, each processor
receives the values i=*, j=*, k=rank, i.e. one z plane.
Note: This code is written assuming only one degree of freedom per node.
For multiple degrees of freedom per node use ISCreateBlock()
instead of ISCreateGeneral().
*/
PetscErrorCode GenerateSliceScatter(DM da,VecScatter *scatter,Vec *vslice)
{
AO ao;
PetscInt M,N,P,nslice,*sliceindices,count,i,j;
PetscMPIInt rank;
PetscErrorCode ierr;
MPI_Comm comm;
Vec vglobal;
IS isfrom,isto;
ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&M,&N,&P,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
/*
nslice is number of degrees of freedom in this processors slice
if there are more processors then z plans the extra processors get 0
elements in their slice.
*/
if (rank < P) nslice = M*N;
else nslice = 0;
/*
Generate the local vector to hold this processors slice
*/
ierr = VecCreateSeq(PETSC_COMM_SELF,nslice,vslice);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&vglobal);CHKERRQ(ierr);
/*
Generate the indices for the slice in the "natural" global ordering
Note: this is just an example, one could select any subset of nodes
on each processor. Just list them in the global natural ordering.
*/
ierr = PetscMalloc1((nslice+1),&sliceindices);CHKERRQ(ierr);
count = 0;
if (rank < P) {
for (j=0; j<N; j++) {
for (i=0; i<M; i++) {
sliceindices[count++] = rank*M*N + j*M + i;
}
}
}
/*
Convert the indices to the "PETSc" global ordering
*/
ierr = AOApplicationToPetsc(ao,nslice,sliceindices);CHKERRQ(ierr);
/* Create the "from" and "to" index set */
/* This is to scatter from the global vector */
ierr = ISCreateGeneral(PETSC_COMM_SELF,nslice,sliceindices,PETSC_OWN_POINTER,&isfrom);CHKERRQ(ierr);
/* This is to gather into the local vector */
ierr = ISCreateStride(PETSC_COMM_SELF,nslice,0,1,&isto);CHKERRQ(ierr);
ierr = VecScatterCreate(vglobal,isfrom,*vslice,isto,scatter);CHKERRQ(ierr);
ierr = ISDestroy(&isfrom);CHKERRQ(ierr);
ierr = ISDestroy(&isto);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode ComputeMapping(DomainData dd,ISLocalToGlobalMapping *isg2lmap)
{
PetscErrorCode ierr;
DM da;
AO ao;
DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE, bz = DM_BOUNDARY_NONE;
DMDAStencilType stype = DMDA_STENCIL_BOX;
ISLocalToGlobalMapping temp_isg2lmap;
PetscInt i,j,k,ig,jg,kg,lindex,gindex,localsize;
PetscInt *global_indices;
PetscFunctionBeginUser;
/* Not an efficient mapping: this function computes a very simple lexicographic mapping
just to illustrate the creation of a MATIS object */
localsize = dd.xm_l*dd.ym_l*dd.zm_l;
ierr = PetscMalloc1(localsize,&global_indices);CHKERRQ(ierr);
for (k=0; k<dd.zm_l; k++) {
kg=dd.startz+k;
for (j=0; j<dd.ym_l; j++) {
jg=dd.starty+j;
for (i=0; i<dd.xm_l; i++) {
ig =dd.startx+i;
lindex =k*dd.xm_l*dd.ym_l+j*dd.xm_l+i;
gindex =kg*dd.xm*dd.ym+jg*dd.xm+ig;
global_indices[lindex]=gindex;
}
}
}
if (dd.dim==3) {
ierr = DMDACreate3d(dd.gcomm,bx,by,bz,stype,dd.xm,dd.ym,dd.zm,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dd.dim==2) {
ierr = DMDACreate2d(dd.gcomm,bx,by,stype,dd.xm,dd.ym,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(dd.gcomm,bx,dd.xm,1,1,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDASetAOType(da,AOMEMORYSCALABLE);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,dd.xm_l*dd.ym_l*dd.zm_l,global_indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(dd.gcomm,1,localsize,global_indices,PETSC_OWN_POINTER,&temp_isg2lmap);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
*isg2lmap = temp_isg2lmap;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt i,n = 5;
PetscInt getpetsc[] = {0,3,4},getapp[] = {2,1,9,7};
PetscInt getpetsc1[] = {0,3,4},getapp1[] = {2,1,9,7};
PetscInt getpetsc2[] = {0,3,4},getapp2[] = {2,1,9,7};
PetscInt getpetsc3[] = {0,3,4},getapp3[] = {2,1,9,7};
PetscInt getpetsc4[] = {0,3,4},getapp4[] = {2,1,9,7};
PetscMPIInt rank,size;
IS ispetsc,isapp;
AO ao;
const PetscInt *app;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/* create the index sets */
ierr = ISCreateStride(PETSC_COMM_WORLD,n,rank,size,&isapp);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_WORLD,n,n*rank,1,&ispetsc);CHKERRQ(ierr); /* natural numbering */
/* create the application ordering */
ierr = AOCreateBasicIS(isapp,ispetsc,&ao);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] 2,1,9,7 PetscToApplication %D %D %D %D\n",rank,getapp[0],getapp[1],getapp[2],getapp[3]);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] 0,3,4 ApplicationToPetsc %D %D %D\n",rank,getpetsc[0],getpetsc[1],getpetsc[2]);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* test MemoryScalable ao */
/*-------------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTest AOCreateMemoryScalable: \n");CHKERRQ(ierr);
ierr = AOCreateMemoryScalableIS(isapp,ispetsc,&ao);CHKERRQ(ierr);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp1);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc1);CHKERRQ(ierr);
/* Check accuracy */;
for (i=0; i<4; i++) {
if (getapp1[i] != getapp[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getapp1 %d != getapp %d",getapp1[i],getapp[i]);
}
for (i=0; i<3; i++) {
if (getpetsc1[i] != getpetsc[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getpetsc1 %d != getpetsc %d",getpetsc1[i],getpetsc[i]);
}
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* test MemoryScalable ao: ispetsc = NULL */
/*-----------------------------------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTest AOCreateMemoryScalable with ispetsc=NULL:\n");CHKERRQ(ierr);
ierr = AOCreateMemoryScalableIS(isapp,NULL,&ao);CHKERRQ(ierr);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp2);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc2);CHKERRQ(ierr);
/* Check accuracy */;
for (i=0; i<4; i++) {
if (getapp2[i] != getapp[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getapp2 %d != getapp %d",getapp2[i],getapp[i]);
}
for (i=0; i<3; i++) {
if (getpetsc2[i] != getpetsc[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getpetsc2 %d != getpetsc %d",getpetsc2[i],getpetsc[i]);
}
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* test AOCreateMemoryScalable() ao: */
ierr = ISGetIndices(isapp,&app);CHKERRQ(ierr);
ierr = AOCreateMemoryScalable(PETSC_COMM_WORLD,n,app,NULL,&ao);CHKERRQ(ierr);
ierr = ISRestoreIndices(isapp,&app);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp4);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc4);CHKERRQ(ierr);
/* Check accuracy */;
for (i=0; i<4; i++) {
if (getapp4[i] != getapp[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getapp4 %d != getapp %d",getapp4[i],getapp[i]);
}
for (i=0; i<3; i++) {
if (getpetsc4[i] != getpetsc[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getpetsc4 %d != getpetsc %d",getpetsc4[i],getpetsc[i]);
}
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* test general API */
/*------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTest general API: \n");CHKERRQ(ierr);
ierr = AOCreate(PETSC_COMM_WORLD,&ao);CHKERRQ(ierr);
ierr = AOSetIS(ao,isapp,ispetsc);CHKERRQ(ierr);
ierr = AOSetType(ao,AOMEMORYSCALABLE);CHKERRQ(ierr);
ierr = AOSetFromOptions(ao);CHKERRQ(ierr);
/* ispetsc and isapp are nolonger used. */
ierr = ISDestroy(&ispetsc);CHKERRQ(ierr);
ierr = ISDestroy(&isapp);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp3);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc3);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] 2,1,9,7 PetscToApplication %D %D %D %D\n",rank,getapp3[0],getapp3[1],getapp3[2],getapp3[3]);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] 0,3,4 ApplicationToPetsc %D %D %D\n",rank,getpetsc3[0],getpetsc3[1],getpetsc3[2]);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
/* Check accuracy */;
for (i=0; i<4; i++) {
if (getapp3[i] != getapp[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getapp3 %d != getapp %d",getapp3[i],getapp[i]);
}
for (i=0; i<3; i++) {
if (getpetsc3[i] != getpetsc[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getpetsc3 %d != getpetsc %d",getpetsc3[i],getpetsc[i]);
}
ierr = AODestroy(&ao);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode vizGA2DA()
{
PetscErrorCode ierr;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
int d1 = 40, d2 = 50;
DA da;
Vec vec;
const PetscInt *lx, *ly, *lz;
PetscInt m,n,p;
DALocalInfo info;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr);
ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr);
/**/
ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
for (int i = 0; i < m; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]);
}
for (int i = 0; i < n; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]);
}
/**/
int ga = GA_Create_handle();
int ndim = 2;
int dims[2] = {d2,d1};
GA_Set_data(ga,2,dims,MT_DBL);
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < n; i++ )
{
map[i] = ly[i-1] + map[i-1];
}
map[n] = 0;
for( int i = n+1; i < m+n; i++ )
{
map[i] = lx[i-n-1] + map[i-1];
}
/* correct ordering, but nodeid's dont line up with mpi rank for petsc's da
* DA: +---+---+ GA: +---+---+
* +-2-+-3-+ +-1-+-3-+
* +---+---+ +---+---+
* +-0-+-1-+ +-0-+-2-+
* +---+---+ +---+---+
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < m; i++ )
{
map[i] = lx[i] + map[i-1];
}
map[m] = 0;
for( int i = m+1; i < m+n; i++ )
{
map[i] = ly[i-m] + map[i-1];
}
*/
int block[2] = {n,m};
GA_Set_irreg_distr(ga,map,block);
ierr = GA_Allocate( ga );
if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr);
if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr);
if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1);
GA_Print_distribution(ga);
int lo[2], hi[2];
NGA_Distribution(ga,rank,lo,hi);
if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 ||
lo[0] != info.ys || hi[0] != info.ys+info.ym-1 )
{
PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n",
rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1);
}
PetscBarrier(0);
PetscSynchronizedFlush(PETSC_COMM_WORLD);
AO ao;
DAGetAO(da,&ao);
if( rank == 0 )
{
int *idx, len = d1*d2;
PetscReal *val;
PetscMalloc(sizeof(PetscReal)*len, &val);
PetscMalloc(sizeof(int)*len, &idx);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
idx[i + d1*j] = i + d1*j;
val[i + d1*j] = i + d1*j;
}
}
AOApplicationToPetsc(ao,len,idx);
VecSetValues(vec,len,idx,val,INSERT_VALUES);
int a[2], b[2],ld[1]={0};
double c = 0;
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
a[0] = j;
a[1] = i;
// printf("%5.0f ",c);
NGA_Put(ga,a,a,&c,ld);
c++;
}
}
}
// GA_Print(ga);
VecAssemblyBegin(vec);
VecAssemblyEnd(vec);
int ld;
double *ptr;
NGA_Access(ga,lo,hi,&ptr,&ld);
PetscReal **d;
int c=0;
ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr);
for (int j = info.ys; j < info.ys+info.ym; ++j)
{
for (int i = info.xs; i < info.xs+info.xm; ++i)
{
if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] )
GA_Error("DA array is not equal to GA array",1);
// printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]);
c++;
}
}
ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr);
c=0;
PetscReal *v;
int start, end;
VecGetOwnershipRange(vec, &start, &end);
VecGetArray( vec, &v );
for( int i = start; i < end; i++)
{
// printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") );
}
VecRestoreArray( vec, &v );
NGA_Release_update(ga,lo,hi);
Vec gada;
VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada);
VecView(gada,PETSC_VIEWER_STDOUT_SELF);
GA_Destroy(ga);
ierr = VecDestroy(vec); CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscErrorCode ierr;
PetscInt M = 10,N = 8,m = PETSC_DECIDE;
PetscInt s =2,w=2,n = PETSC_DECIDE,nloc,l,i,j,kk;
PetscInt Xs,Xm,Ys,Ym,iloc,*iglobal;
const PetscInt *ltog;
PetscInt *lx = NULL,*ly = NULL;
PetscBool testorder = PETSC_FALSE,flg;
DMBoundaryType bx = DM_BOUNDARY_NONE,by= DM_BOUNDARY_NONE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDAStencilType st = DMDA_STENCIL_BOX;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,400,&viewer);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(NULL,NULL,"-NX",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-NY",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-w",&w,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xghosted",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yghosted",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-star",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-box",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-testorder",&testorder,NULL);CHKERRQ(ierr);
/*
Test putting two nodes in x and y on each processor, exact last processor
in x and y gets the rest.
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-distribute",&flg,NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,st,M,N,m,n,w,s,lx,ly,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!testorder) { /* turn off printing when testing ordering mappings */
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vectors:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n");CHKERRQ(ierr);
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (testorder) {
ISLocalToGlobalMapping ltogm;
ierr = DMGetLocalToGlobalMapping(da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltogm,&nloc);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,NULL,&Xm,&Ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = PetscMalloc1(nloc,&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscFPrintf(PETSC_COMM_SELF,stdout,"[%d] Problem with mapping: j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",rank,j,i,l,ltog[iloc+l],iglobal[kk]);CHKERRQ(ierr);
}
kk++;
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode FACreate(FA *infa)
{
FA fa;
PetscMPIInt rank;
PetscInt tonglobal,globalrstart,x,nx,y,ny,*tonatural,i,j,*to,*from,offt[3];
PetscInt *fromnatural,fromnglobal,nscat,nlocal,cntl1,cntl2,cntl3,*indices;
PetscErrorCode ierr;
/* Each DMDA manages the local vector for the portion of region 1, 2, and 3 for that processor
Each DMDA can belong on any subset (overlapping between DMDA's or not) of processors
For processes that a particular DMDA does not exist on, the corresponding comm should be set to zero
*/
DM da1 = 0,da2 = 0,da3 = 0;
/*
v1, v2, v3 represent the local vector for a single DMDA
*/
Vec vl1 = 0,vl2 = 0,vl3 = 0, vg1 = 0, vg2 = 0,vg3 = 0;
/*
globalvec and friends represent the global vectors that are used for the PETSc solvers
localvec represents the concatenation of the (up to) 3 local vectors; vl1, vl2, vl3
tovec and friends represent intermediate vectors that are ONLY used for setting up the
final communication patterns. Once this setup routine is complete they are destroyed.
The tovec is like the globalvec EXCEPT it has redundant locations for the ghost points
between regions 2+3 and 1.
*/
AO toao,globalao;
IS tois,globalis,is;
Vec tovec,globalvec,localvec;
VecScatter vscat;
PetscScalar *globalarray,*localarray,*toarray;
ierr = PetscNew(struct _p_FA,&fa);CHKERRQ(ierr);
/*
fa->sw is the stencil width
fa->p1 is the width of region 1, fa->p2 the width of region 2 (must be the same)
fa->r1 height of region 1
fa->r2 height of region 2
fa->r2 is also the height of region 3-4
(fa->p1 - fa->p2)/2 is the width of both region 3 and region 4
*/
fa->p1 = 24;
fa->p2 = 15;
fa->r1 = 6;
fa->r2 = 6;
fa->sw = 1;
fa->r1g = fa->r1 + fa->sw;
fa->r2g = fa->r2 + fa->sw;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
fa->comm[0] = PETSC_COMM_WORLD;
fa->comm[1] = PETSC_COMM_WORLD;
fa->comm[2] = PETSC_COMM_WORLD;
/* Test case with different communicators */
/* Normally one would use MPI_Comm routines to build MPI communicators on which you wish to partition the DMDAs*/
/*
if (!rank) {
fa->comm[0] = PETSC_COMM_SELF;
fa->comm[1] = 0;
fa->comm[2] = 0;
} else if (rank == 1) {
fa->comm[0] = 0;
fa->comm[1] = PETSC_COMM_SELF;
fa->comm[2] = 0;
} else {
fa->comm[0] = 0;
fa->comm[1] = 0;
fa->comm[2] = PETSC_COMM_SELF;
} */
if (fa->p2 > fa->p1 - 3) SETERRQ(PETSC_COMM_SELF,1,"Width of region fa->p2 must be at least 3 less then width of region 1");
if (!((fa->p2 - fa->p1) % 2)) SETERRQ(PETSC_COMM_SELF,1,"width of region 3 must NOT be divisible by 2");
if (fa->comm[1]) {
ierr = DMDACreate2d(fa->comm[1],DMDA_BOUNDARY_PERIODIC,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p2,fa->r2g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da2);CHKERRQ(ierr);
ierr = DMGetLocalVector(da2,&vl2);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da2,&vg2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = DMDACreate2d(fa->comm[2],DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p1-fa->p2,fa->r2g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da3);CHKERRQ(ierr);
ierr = DMGetLocalVector(da3,&vl3);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da3,&vg3);CHKERRQ(ierr);
}
if (fa->comm[0]) {
ierr = DMDACreate2d(fa->comm[0],DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p1,fa->r1g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da1);CHKERRQ(ierr);
ierr = DMGetLocalVector(da1,&vl1);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da1,&vg1);CHKERRQ(ierr);
}
/* count the number of unknowns owned on each processor and determine the starting point of each processors ownership
for global vector with redundancy */
tonglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Number of unknowns owned %d\n",rank,tonglobal);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
/* Get tonatural number for each node */
ierr = PetscMalloc((tonglobal+1)*sizeof(PetscInt),&tonatural);CHKERRQ(ierr);
tonglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
tonatural[tonglobal++] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) tonatural[tonglobal++] = x + i + fa->p1*(y + j);
else tonatural[tonglobal++] = fa->p2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
tonatural[tonglobal++] = fa->p1*fa->r2g + x + i + fa->p1*(y + j);
}
}
}
/* ierr = PetscIntView(tonglobal,tonatural,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = AOCreateBasic(PETSC_COMM_WORLD,tonglobal,tonatural,0,&toao);CHKERRQ(ierr);
ierr = PetscFree(tonatural);CHKERRQ(ierr);
/* count the number of unknowns owned on each processor and determine the starting point of each processors ownership
for global vector without redundancy */
fromnglobal = 0;
fa->offg[1] = 0;
offt[1] = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
offt[2] = nx*ny;
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fromnglobal += nx*ny;
fa->offg[2] = fromnglobal;
} else {
offt[2] = 0;
fa->offg[2] = 0;
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
offt[0] = offt[2] + nx*ny;
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fromnglobal += nx*ny;
fa->offg[0] = fromnglobal;
} else {
offt[0] = offt[2];
fa->offg[0] = fromnglobal;
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y == 0) ny--; /* includes the ghost points on the lower side */
fromnglobal += nx*ny;
}
ierr = MPI_Scan(&fromnglobal,&globalrstart,1,MPIU_INT,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
globalrstart -= fromnglobal;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Number of unknowns owned %d\n",rank,fromnglobal);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
/* Get fromnatural number for each node */
ierr = PetscMalloc((fromnglobal+1)*sizeof(PetscInt),&fromnatural);CHKERRQ(ierr);
fromnglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fa->xg[1] = x; fa->yg[1] = y; fa->mg[1] = nx; fa->ng[1] = ny;
ierr = DMDAGetGhostCorners(da2,&fa->xl[1],&fa->yl[1],0,&fa->ml[1],&fa->nl[1],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
fromnatural[fromnglobal++] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fa->xg[2] = x; fa->yg[2] = y; fa->mg[2] = nx; fa->ng[2] = ny;
ierr = DMDAGetGhostCorners(da3,&fa->xl[2],&fa->yl[2],0,&fa->ml[2],&fa->nl[2],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) fromnatural[fromnglobal++] = x + i + fa->p1*(y + j);
else fromnatural[fromnglobal++] = fa->p2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y == 0) ny--; /* includes the ghost points on the lower side */
else y--;
fa->xg[0] = x; fa->yg[0] = y; fa->mg[0] = nx; fa->ng[0] = ny;
ierr = DMDAGetGhostCorners(da1,&fa->xl[0],&fa->yl[0],0,&fa->ml[0],&fa->nl[0],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
fromnatural[fromnglobal++] = fa->p1*fa->r2 + x + i + fa->p1*(y + j);
}
}
}
/*ierr = PetscIntView(fromnglobal,fromnatural,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
ierr = AOCreateBasic(PETSC_COMM_WORLD,fromnglobal,fromnatural,0,&globalao);CHKERRQ(ierr);
ierr = PetscFree(fromnatural);CHKERRQ(ierr);
/* ---------------------------------------------------*/
/* Create the scatter that updates 1 from 2 and 3 and 3 and 2 from 1 */
/* currently handles stencil width of 1 ONLY */
ierr = PetscMalloc(tonglobal*sizeof(PetscInt),&to);CHKERRQ(ierr);
ierr = PetscMalloc(tonglobal*sizeof(PetscInt),&from);CHKERRQ(ierr);
nscat = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
to[nscat] = from[nscat] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);nscat++;
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) {
to[nscat] = from[nscat] = x + i + fa->p1*(y + j);nscat++;
} else {
to[nscat] = from[nscat] = fa->p2 + x + i + fa->p1*(y + j);nscat++;
}
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
to[nscat] = fa->p1*fa->r2g + x + i + fa->p1*(y + j);
from[nscat++] = fa->p1*(fa->r2 - 1) + x + i + fa->p1*(y + j);
}
}
}
ierr = AOApplicationToPetsc(toao,nscat,to);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(globalao,nscat,from);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,nscat,to,PETSC_COPY_VALUES,&tois);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,nscat,from,PETSC_COPY_VALUES,&globalis);CHKERRQ(ierr);
ierr = PetscFree(to);CHKERRQ(ierr);
ierr = PetscFree(from);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,tonglobal,PETSC_DETERMINE,&tovec);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,fromnglobal,PETSC_DETERMINE,&globalvec);CHKERRQ(ierr);
ierr = VecScatterCreate(globalvec,globalis,tovec,tois,&vscat);CHKERRQ(ierr);
ierr = ISDestroy(&tois);CHKERRQ(ierr);
ierr = ISDestroy(&globalis);CHKERRQ(ierr);
ierr = AODestroy(&globalao);CHKERRQ(ierr);
ierr = AODestroy(&toao);CHKERRQ(ierr);
/* fill up global vector without redundant values with PETSc global numbering */
ierr = VecGetArray(globalvec,&globalarray);CHKERRQ(ierr);
for (i=0; i<fromnglobal; i++) {
globalarray[i] = globalrstart + i;
}
ierr = VecRestoreArray(globalvec,&globalarray);CHKERRQ(ierr);
/* scatter PETSc global indices to redundant valueed array */
ierr = VecScatterBegin(vscat,globalvec,tovec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,globalvec,tovec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Create local vector that is the concatenation of the local vectors */
nlocal = 0;
cntl1 = cntl2 = cntl3 = 0;
if (fa->comm[1]) {
ierr = VecGetSize(vl2,&cntl2);CHKERRQ(ierr);
nlocal += cntl2;
}
if (fa->comm[2]) {
ierr = VecGetSize(vl3,&cntl3);CHKERRQ(ierr);
nlocal += cntl3;
}
if (fa->comm[0]) {
ierr = VecGetSize(vl1,&cntl1);CHKERRQ(ierr);
nlocal += cntl1;
}
fa->offl[0] = cntl2 + cntl3;
fa->offl[1] = 0;
fa->offl[2] = cntl2;
ierr = VecCreateSeq(PETSC_COMM_SELF,nlocal,&localvec);CHKERRQ(ierr);
/* cheat so that vl1, vl2, vl3 shared array memory with localvec */
ierr = VecGetArray(localvec,&localarray);CHKERRQ(ierr);
ierr = VecGetArray(tovec,&toarray);CHKERRQ(ierr);
if (fa->comm[1]) {
ierr = VecPlaceArray(vl2,localarray+fa->offl[1]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg2,toarray+offt[1]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da2,vg2,INSERT_VALUES,vl2);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da2,vg2,INSERT_VALUES,vl2);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da2,&vg2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = VecPlaceArray(vl3,localarray+fa->offl[2]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg3,toarray+offt[2]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da3,vg3,INSERT_VALUES,vl3);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da3,vg3,INSERT_VALUES,vl3);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da3,&vg3);CHKERRQ(ierr);
}
if (fa->comm[0]) {
ierr = VecPlaceArray(vl1,localarray+fa->offl[0]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg1,toarray+offt[0]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da1,vg1,INSERT_VALUES,vl1);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da1,vg1,INSERT_VALUES,vl1);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da1,&vg1);CHKERRQ(ierr);
}
ierr = VecRestoreArray(localvec,&localarray);CHKERRQ(ierr);
ierr = VecRestoreArray(tovec,&toarray);CHKERRQ(ierr);
/* no longer need the redundant vector and VecScatter to it */
ierr = VecScatterDestroy(&vscat);CHKERRQ(ierr);
ierr = VecDestroy(&tovec);CHKERRQ(ierr);
/* Create final scatter that goes directly from globalvec to localvec */
/* this is the one to be used in the application code */
ierr = PetscMalloc(nlocal*sizeof(PetscInt),&indices);CHKERRQ(ierr);
ierr = VecGetArray(localvec,&localarray);CHKERRQ(ierr);
for (i=0; i<nlocal; i++) {
indices[i] = (PetscInt) (localarray[i]);
}
ierr = VecRestoreArray(localvec,&localarray);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_WORLD,2,nlocal,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = PetscFree(indices);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,2*nlocal,&fa->l);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,2*fromnglobal,PETSC_DETERMINE,&fa->g);CHKERRQ(ierr);
ierr = VecScatterCreate(fa->g,is,fa->l,NULL,&fa->vscat);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = VecDestroy(&globalvec);CHKERRQ(ierr);
ierr = VecDestroy(&localvec);CHKERRQ(ierr);
if (fa->comm[0]) {
ierr = DMRestoreLocalVector(da1,&vl1);CHKERRQ(ierr);
ierr = DMDestroy(&da1);CHKERRQ(ierr);
}
if (fa->comm[1]) {
ierr = DMRestoreLocalVector(da2,&vl2);CHKERRQ(ierr);
ierr = DMDestroy(&da2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = DMRestoreLocalVector(da3,&vl3);CHKERRQ(ierr);
ierr = DMDestroy(&da3);CHKERRQ(ierr);
}
*infa = fa;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 3,N = 5,P=3,s=1,w=2,nloc,l,i,j,k,kk,m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE;
PetscErrorCode ierr;
PetscInt Xs,Xm,Ys,Ym,Zs,Zm,iloc,*ltog,*iglobal;
PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL,*lz = PETSC_NULL;
PetscBool test_order = PETSC_FALSE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
AO ao;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,300,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-NX",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NY",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NZ",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-w",&w,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-box",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xnonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-ynonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-znonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-testorder",&test_order,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-distribute",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc(m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc(n*sizeof(PetscInt),&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
if (p == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -p option with -distribute option");
ierr = PetscMalloc(p*sizeof(PetscInt),&lz);CHKERRQ(ierr);
for (i=0; i<p-1; i++) { lz[i] = 2;}
lz[p-1] = P - 2*(p-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,m,n,p,w,s,lx,ly,lz,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = PetscFree(lz);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!test_order) { /* turn off printing when testing ordering mappings */
if (M*N*P<40) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-local_print",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (test_order) {
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,&Zs,&Xm,&Ym,&Zm);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(da,&nloc,<og);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
/* ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = PetscMalloc(nloc*sizeof(PetscInt),&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y, then z) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscPrintf(MPI_COMM_WORLD,"[%D] Problem with mapping: z=%D, j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",
rank,k,j,i,l,ltog[iloc+l],iglobal[kk]);
}
kk++;
}
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int MeshData::get_AppToPETSc_Ordering(int n) const {
AOApplicationToPetsc(ao,1,&n);
return n;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt rank;
PetscInt i, j;
PetscInt um, un, vm, vn, pm, pn;
PetscInt umstart, unstart, vmstart, vnstart;
PetscInt *d_nnz, *o_nnz;
PetscInt uStart, uEnd, pStart, pEnd, uLocalSize, pLocalSize;
PetscInt col;
//PetscInt row, cols[2];
PetscInt localIdx;
AO pao;
//PetscScalar values[2];
//IS *is;
//const PetscInt *xindices, *yindices;
simInfo data;
ierr = PetscInitialize(&argc, &argv, NULL, NULL); CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
// defaults
data.nx = 5;
data.ny = 5;
// Read options
ierr = PetscOptionsGetInt(NULL, "-nx", &(data.nx), NULL); CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL, "-ny", &(data.ny), NULL); CHKERRQ(ierr);
hline();
ierr = PetscPrintf(PETSC_COMM_WORLD, "Program that reminds you why you need to use AOApplicationToPetsc if you want to map from P to UV"); CHKERRQ(ierr);
hline();
createArrays(&data);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nCreate matrix G that maps from P to U [VecGetOwnershipRange, MatCreateAIJ]"); CHKERRQ(ierr); hline();
// ownership range of u
ierr = VecGetOwnershipRange(data.uPacked, &uStart, &uEnd); CHKERRQ(ierr);
uLocalSize = uEnd-uStart;
// create arrays to store nnz values
ierr = PetscMalloc(uLocalSize*sizeof(PetscInt), &d_nnz); CHKERRQ(ierr);
ierr = PetscMalloc(uLocalSize*sizeof(PetscInt), &o_nnz); CHKERRQ(ierr);
// ownership range of phi
ierr = VecGetOwnershipRange(data.pGlobal, &pStart, &pEnd); CHKERRQ(ierr);
pLocalSize = pEnd-pStart;
ierr = DMDAGetCorners(data.pda, NULL, NULL, NULL, &pm, &pn, NULL); CHKERRQ(ierr);
// count the number of non-zeros in each row
ierr = DMDAGetAO(data.pda, &pao); CHKERRQ(ierr);
ierr = DMDAGetCorners(data.uda, &umstart, &unstart, NULL, &um, &un, NULL); CHKERRQ(ierr);
localIdx = 0;
for(j=unstart; j<unstart+un; j++)
{
for(i=umstart; i<umstart+um; i++)
{
d_nnz[localIdx] = 0;
o_nnz[localIdx] = 0;
// G portion
col = j*data.nx+i;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
col = pStart + (j-unstart)*pm + i-umstart;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\t", col); CHKERRQ(ierr);
(col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++;
col = j*data.nx+i+1;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
col = pStart + (j-unstart)*pm + i+1-umstart;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\n", col); CHKERRQ(ierr);
(col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++;
localIdx++;
}
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "-\n"); CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr);
ierr = DMDAGetCorners(data.vda, &vmstart, &vnstart, NULL, &vm, &vn, NULL); CHKERRQ(ierr);
for(j=vnstart; j<vnstart+vn; j++)
{
for(i=vmstart; i<vmstart+vm; i++)
{
d_nnz[localIdx] = 0;
o_nnz[localIdx] = 0;
// G portion
col = j*data.nx+i;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
col = pStart + (j-unstart)*pm + i-umstart;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\t", col); CHKERRQ(ierr);
(col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++;
col = (j+1)*data.nx+i;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
col = pStart + (j+1-unstart)*pm + i-umstart;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\n", col); CHKERRQ(ierr);
(col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++;
localIdx++;
}
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "-\n"); CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr);
//ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%d + %d = %d\n", m*n, p*q, uLocalSize); CHKERRQ(ierr);
//ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr);
/*
ierr = MatCreateAIJ(PETSC_COMM_WORLD, uLocalSize, pLocalSize, PETSC_DETERMINE, PETSC_DETERMINE, 2, NULL, 1, NULL, &G); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nPopulate G [DMCompositeGetGlobalISs, ISGetIndices, DMDAGetCorners, AOApplicationToPetsc, MatSetValues, MatAssemblyBegin, MatAssemblyEnd]"); CHKERRQ(ierr); hline();
ierr = DMCompositeGetGlobalISs(pack, &is); CHKERRQ(ierr);
ierr = ISGetIndices(is[0], &xindices); CHKERRQ(ierr);
ierr = DMDAGetCorners(uda, &mstart, &nstart, NULL, &m, &n, NULL);
localIdx = 0;
for(j=nstart; j<nstart+n; j++)
{
for(i=mstart; i<mstart+m; i++)
{
row = xindices[localIdx]; // uses the indices obtained using DMCompositeGetGlobalISs and ISGetIndices
cols[0] = j*nx+i+1; // i and j are obtained from DMDAGetCorners
cols[1] = j*nx+i;
ierr = AOApplicationToPetsc(pao, 2, cols); CHKERRQ(ierr); // maps the natural ordering to petsc ordering using the AO obtained from DMDAGetAO
values[0] = 1;
values[1] = -1;
ierr = MatSetValues(G, 1, &row, 2, cols, values, INSERT_VALUES); CHKERRQ(ierr);
localIdx++;
}
}
ierr = ISRestoreIndices(is[0], &xindices); CHKERRQ(ierr);
ierr = ISGetIndices(is[1], &yindices); CHKERRQ(ierr);
ierr = DMDAGetCorners(vda, &mstart, &nstart, NULL, &m, &n, NULL);
localIdx = 0;
for(j=nstart; j<nstart+n; j++)
{
for(i=mstart; i<mstart+m; i++)
{
row = yindices[localIdx];
cols[0] = (j+1)*nx+i;
cols[1] = j*nx+i;
ierr = AOApplicationToPetsc(pao, 2, cols); CHKERRQ(ierr);
values[0] = 1;
values[1] = -1;
ierr = MatSetValues(G, 1, &row, 2, cols, values, INSERT_VALUES); CHKERRQ(ierr);
localIdx++;
}
}
ierr = ISRestoreIndices(is[1], &yindices); CHKERRQ(ierr);
ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatView(G, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nU = G*P [MatMult]"); CHKERRQ(ierr); hline();
ierr = MatMult(G, pGlobal, uPacked); CHKERRQ(ierr);
ierr = VecView(uPacked, PETSC_VIEWER_STDOUT_WORLD);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nC = GT*G [MatTransposeMatMult]"); CHKERRQ(ierr); hline();
ierr = MatTransposeMatMult(G, G, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &C); CHKERRQ(ierr);
ierr = MatView(C, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nbc2 = C*P [MatMult]"); CHKERRQ(ierr); hline();
ierr = MatMult(C, pGlobal, bc2); CHKERRQ(ierr);
ierr = VecView(bc2, PETSC_VIEWER_STDOUT_WORLD);
ierr = MatDestroy(&C); CHKERRQ(ierr);
ierr = MatDestroy(&G); CHKERRQ(ierr);
ierr = ISDestroy(&is[0]); CHKERRQ(ierr);
ierr = ISDestroy(&is[1]); CHKERRQ(ierr);
ierr = PetscFree(is); CHKERRQ(ierr);
ierr = VecDestroy(&bc2); CHKERRQ(ierr);
ierr = VecDestroy(&uPacked); CHKERRQ(ierr);
ierr = VecDestroy(&pGlobal); CHKERRQ(ierr);
ierr = VecDestroy(&bc2); CHKERRQ(ierr);
ierr = DMDestroy(&vda); CHKERRQ(ierr);
ierr = DMDestroy(&uda); CHKERRQ(ierr);
ierr = DMDestroy(&pda); CHKERRQ(ierr);
ierr = DMDestroy(&pack); CHKERRQ(ierr);
*/
destroyArrays(&data);
ierr = PetscFinalize();
return 0;
}
