/*
Given a DMDA generates a VecScatter context that will deliver a slice
of the global vector to each processor. In this example, each processor
receives the values i=*, j=*, k=rank, i.e. one z plane.
Note: This code is written assuming only one degree of freedom per node.
For multiple degrees of freedom per node use ISCreateBlock()
instead of ISCreateGeneral().
*/
PetscErrorCode GenerateSliceScatter(DM da,VecScatter *scatter,Vec *vslice)
{
AO ao;
PetscInt M,N,P,nslice,*sliceindices,count,i,j;
PetscMPIInt rank;
PetscErrorCode ierr;
MPI_Comm comm;
Vec vglobal;
IS isfrom,isto;
ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&M,&N,&P,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
/*
nslice is number of degrees of freedom in this processors slice
if there are more processors then z plans the extra processors get 0
elements in their slice.
*/
if (rank < P) nslice = M*N;
else nslice = 0;
/*
Generate the local vector to hold this processors slice
*/
ierr = VecCreateSeq(PETSC_COMM_SELF,nslice,vslice);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&vglobal);CHKERRQ(ierr);
/*
Generate the indices for the slice in the "natural" global ordering
Note: this is just an example, one could select any subset of nodes
on each processor. Just list them in the global natural ordering.
*/
ierr = PetscMalloc1((nslice+1),&sliceindices);CHKERRQ(ierr);
count = 0;
if (rank < P) {
for (j=0; j<N; j++) {
for (i=0; i<M; i++) {
sliceindices[count++] = rank*M*N + j*M + i;
}
}
}
/*
Convert the indices to the "PETSc" global ordering
*/
ierr = AOApplicationToPetsc(ao,nslice,sliceindices);CHKERRQ(ierr);
/* Create the "from" and "to" index set */
/* This is to scatter from the global vector */
ierr = ISCreateGeneral(PETSC_COMM_SELF,nslice,sliceindices,PETSC_OWN_POINTER,&isfrom);CHKERRQ(ierr);
/* This is to gather into the local vector */
ierr = ISCreateStride(PETSC_COMM_SELF,nslice,0,1,&isto);CHKERRQ(ierr);
ierr = VecScatterCreate(vglobal,isfrom,*vslice,isto,scatter);CHKERRQ(ierr);
ierr = ISDestroy(&isfrom);CHKERRQ(ierr);
ierr = ISDestroy(&isto);CHKERRQ(ierr);
return 0;
}
/*@
DMDANaturalAllToGlobalCreate - Creates a scatter context that maps from a copy
of the entire vector on each processor to its local part in the global vector.
Collective on DMDA
Input Parameter:
. da - the distributed array context
Output Parameter:
. scatter - the scatter context
Level: advanced
.keywords: distributed array, global to local, begin, coarse problem
.seealso: DMDAGlobalToNaturalEnd(), DMLocalToGlobalBegin(), DMDACreate2d(),
DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector()
@*/
PetscErrorCode DMDANaturalAllToGlobalCreate(DM da,VecScatter *scatter)
{
PetscErrorCode ierr;
DM_DA *dd = (DM_DA*)da->data;
PetscInt M,m = dd->Nlocal,start;
IS from,to;
Vec tmplocal,global;
AO ao;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidPointer(scatter,2);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
/* create the scatter context */
ierr = MPI_Allreduce(&m,&M,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)da));CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)da),dd->w,m,PETSC_DETERMINE,0,&global);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(global,&start,NULL);CHKERRQ(ierr);
ierr = ISCreateStride(PetscObjectComm((PetscObject)da),m,start,1,&from);CHKERRQ(ierr);
ierr = AOPetscToApplicationIS(ao,from);CHKERRQ(ierr);
ierr = ISCreateStride(PetscObjectComm((PetscObject)da),m,start,1,&to);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,dd->w,M,0,&tmplocal);CHKERRQ(ierr);
ierr = VecScatterCreate(tmplocal,from,global,to,scatter);CHKERRQ(ierr);
ierr = VecDestroy(&tmplocal);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMDAGlobalToNaturalAllCreate - Creates a scatter context that maps from the
global vector the entire vector to each processor in natural numbering
Collective on DMDA
Input Parameter:
. da - the distributed array context
Output Parameter:
. scatter - the scatter context
Level: advanced
.keywords: distributed array, global to local, begin, coarse problem
.seealso: DMDAGlobalToNaturalEnd(), DMLocalToGlobalBegin(), DMDACreate2d(),
DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector()
@*/
PetscErrorCode DMDAGlobalToNaturalAllCreate(DM da,VecScatter *scatter)
{
PetscErrorCode ierr;
PetscInt N;
IS from,to;
Vec tmplocal,global;
AO ao;
DM_DA *dd = (DM_DA*)da->data;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidPointer(scatter,2);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
/* create the scatter context */
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)da),dd->w,dd->Nlocal,PETSC_DETERMINE,0,&global);CHKERRQ(ierr);
ierr = VecGetSize(global,&N);CHKERRQ(ierr);
ierr = ISCreateStride(PetscObjectComm((PetscObject)da),N,0,1,&to);CHKERRQ(ierr);
ierr = AOPetscToApplicationIS(ao,to);CHKERRQ(ierr);
ierr = ISCreateStride(PetscObjectComm((PetscObject)da),N,0,1,&from);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,dd->w,N,0,&tmplocal);CHKERRQ(ierr);
ierr = VecScatterCreate(global,from,tmplocal,to,scatter);CHKERRQ(ierr);
ierr = VecDestroy(&tmplocal);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMDAGetRay - Returns a vector on process zero that contains a row or column of the values in a DMDA vector
Collective on DMDA
Input Parameters:
+ da - the distributed array
. vec - the vector
. dir - Cartesian direction, either DMDA_X, DMDA_Y, or DMDA_Z
- gp - global grid point number in this direction
Output Parameters:
+ newvec - the new vector that can hold the values (size zero on all processes except process 0)
- scatter - the VecScatter that will map from the original vector to the slice
Level: advanced
Notes:
All processors that share the DMDA must call this with the same gp value
.keywords: distributed array, get, processor subset
@*/
PetscErrorCode DMDAGetRay(DM da,DMDADirection dir,PetscInt gp,Vec *newvec,VecScatter *scatter)
{
PetscMPIInt rank;
DM_DA *dd = (DM_DA*)da->data;
PetscErrorCode ierr;
IS is;
AO ao;
Vec vec;
PetscInt *indices,i,j;
PetscFunctionBegin;
if (dd->dim == 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Cannot get slice from 1d DMDA");
if (dd->dim == 3) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Cannot get slice from 3d DMDA");
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)da),&rank);CHKERRQ(ierr);
if (!rank) {
if (dir == DMDA_Y) {
ierr = PetscMalloc(dd->w*dd->M*sizeof(PetscInt),&indices);CHKERRQ(ierr);
indices[0] = gp*dd->M*dd->w;
for (i=1; i<dd->M*dd->w; i++) indices[i] = indices[i-1] + 1;
ierr = AOApplicationToPetsc(ao,dd->M*dd->w,indices);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,newvec);CHKERRQ(ierr);
ierr = VecSetBlockSize(*newvec,dd->w);CHKERRQ(ierr);
ierr = VecSetSizes(*newvec,dd->M*dd->w,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(*newvec,VECSEQ);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,dd->w*dd->M,indices,PETSC_OWN_POINTER,&is);CHKERRQ(ierr);
} else if (dir == DMDA_X) {
ierr = PetscMalloc(dd->w*dd->N*sizeof(PetscInt),&indices);CHKERRQ(ierr);
indices[0] = dd->w*gp;
for (j=1; j<dd->w; j++) indices[j] = indices[j-1] + 1;
for (i=1; i<dd->N; i++) {
indices[i*dd->w] = indices[i*dd->w-1] + dd->w*dd->M - dd->w + 1;
for (j=1; j<dd->w; j++) indices[i*dd->w + j] = indices[i*dd->w + j - 1] + 1;
}
ierr = AOApplicationToPetsc(ao,dd->w*dd->N,indices);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,newvec);CHKERRQ(ierr);
ierr = VecSetBlockSize(*newvec,dd->w);CHKERRQ(ierr);
ierr = VecSetSizes(*newvec,dd->N*dd->w,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(*newvec,VECSEQ);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,dd->w*dd->N,indices,PETSC_OWN_POINTER,&is);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Unknown DMDADirection");
} else {
ierr = VecCreateSeq(PETSC_COMM_SELF,0,newvec);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,0,0,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
}
ierr = DMGetGlobalVector(da,&vec);CHKERRQ(ierr);
ierr = VecScatterCreate(vec,is,*newvec,NULL,scatter);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&vec);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ComputeMapping(DomainData dd,ISLocalToGlobalMapping *isg2lmap)
{
PetscErrorCode ierr;
DM da;
AO ao;
DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE, bz = DM_BOUNDARY_NONE;
DMDAStencilType stype = DMDA_STENCIL_BOX;
ISLocalToGlobalMapping temp_isg2lmap;
PetscInt i,j,k,ig,jg,kg,lindex,gindex,localsize;
PetscInt *global_indices;
PetscFunctionBeginUser;
/* Not an efficient mapping: this function computes a very simple lexicographic mapping
just to illustrate the creation of a MATIS object */
localsize = dd.xm_l*dd.ym_l*dd.zm_l;
ierr = PetscMalloc1(localsize,&global_indices);CHKERRQ(ierr);
for (k=0; k<dd.zm_l; k++) {
kg=dd.startz+k;
for (j=0; j<dd.ym_l; j++) {
jg=dd.starty+j;
for (i=0; i<dd.xm_l; i++) {
ig =dd.startx+i;
lindex =k*dd.xm_l*dd.ym_l+j*dd.xm_l+i;
gindex =kg*dd.xm*dd.ym+jg*dd.xm+ig;
global_indices[lindex]=gindex;
}
}
}
if (dd.dim==3) {
ierr = DMDACreate3d(dd.gcomm,bx,by,bz,stype,dd.xm,dd.ym,dd.zm,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dd.dim==2) {
ierr = DMDACreate2d(dd.gcomm,bx,by,stype,dd.xm,dd.ym,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(dd.gcomm,bx,dd.xm,1,1,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDASetAOType(da,AOMEMORYSCALABLE);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,dd.xm_l*dd.ym_l*dd.zm_l,global_indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(dd.gcomm,1,localsize,global_indices,PETSC_OWN_POINTER,&temp_isg2lmap);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
*isg2lmap = temp_isg2lmap;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscErrorCode ierr;
PetscInt M = 10,N = 8,m = PETSC_DECIDE;
PetscInt s =2,w=2,n = PETSC_DECIDE,nloc,l,i,j,kk;
PetscInt Xs,Xm,Ys,Ym,iloc,*iglobal;
const PetscInt *ltog;
PetscInt *lx = NULL,*ly = NULL;
PetscBool testorder = PETSC_FALSE,flg;
DMBoundaryType bx = DM_BOUNDARY_NONE,by= DM_BOUNDARY_NONE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDAStencilType st = DMDA_STENCIL_BOX;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,400,&viewer);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(NULL,NULL,"-NX",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-NY",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-w",&w,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xghosted",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yghosted",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-star",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-box",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-testorder",&testorder,NULL);CHKERRQ(ierr);
/*
Test putting two nodes in x and y on each processor, exact last processor
in x and y gets the rest.
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-distribute",&flg,NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,st,M,N,m,n,w,s,lx,ly,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!testorder) { /* turn off printing when testing ordering mappings */
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vectors:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n");CHKERRQ(ierr);
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (testorder) {
ISLocalToGlobalMapping ltogm;
ierr = DMGetLocalToGlobalMapping(da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltogm,&nloc);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,NULL,&Xm,&Ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = PetscMalloc1(nloc,&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscFPrintf(PETSC_COMM_SELF,stdout,"[%d] Problem with mapping: j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",rank,j,i,l,ltog[iloc+l],iglobal[kk]);CHKERRQ(ierr);
}
kk++;
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscInt M = 3,N = 5,P=3,s=1,w=2,nloc,l,i,j,k,kk,m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE;
PetscErrorCode ierr;
PetscInt Xs,Xm,Ys,Ym,Zs,Zm,iloc,*ltog,*iglobal;
PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL,*lz = PETSC_NULL;
PetscBool test_order = PETSC_FALSE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_BOX;
AO ao;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,300,&viewer);CHKERRQ(ierr);
/* Read options */
ierr = PetscOptionsGetInt(PETSC_NULL,"-NX",&M,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NY",&N,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-NZ",&P,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-w",&w,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-box",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) stencil_type = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bx = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-xnonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-yghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) by = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-ynonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zperiodic",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-zghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) bz = DMDA_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-znonghosted",&flg,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(PETSC_NULL,"-testorder",&test_order,PETSC_NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-distribute",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc(m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc(n*sizeof(PetscInt),&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
if (p == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -p option with -distribute option");
ierr = PetscMalloc(p*sizeof(PetscInt),&lz);CHKERRQ(ierr);
for (i=0; i<p-1; i++) { lz[i] = 2;}
lz[p-1] = P - 2*(p-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,m,n,p,w,s,lx,ly,lz,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = PetscFree(lz);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!test_order) { /* turn off printing when testing ordering mappings */
if (M*N*P<40) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-local_print",&flg,PETSC_NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (test_order) {
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,&Zs,&Xm,&Ym,&Zm);CHKERRQ(ierr);
ierr = DMDAGetGlobalIndices(da,&nloc,<og);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
/* ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = PetscMalloc(nloc*sizeof(PetscInt),&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y, then z) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (k=Zs; k<Zs+Zm; k++) {
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscPrintf(MPI_COMM_WORLD,"[%D] Problem with mapping: z=%D, j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",
rank,k,j,i,l,ltog[iloc+l],iglobal[kk]);
}
kk++;
}
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt rank;
PetscInt i, j;
PetscInt um, un, vm, vn, pm, pn;
PetscInt umstart, unstart, vmstart, vnstart;
PetscInt *d_nnz, *o_nnz;
PetscInt uStart, uEnd, pStart, pEnd, uLocalSize, pLocalSize;
PetscInt col;
//PetscInt row, cols[2];
PetscInt localIdx;
AO pao;
//PetscScalar values[2];
//IS *is;
//const PetscInt *xindices, *yindices;
simInfo data;
ierr = PetscInitialize(&argc, &argv, NULL, NULL); CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr);
// defaults
data.nx = 5;
data.ny = 5;
// Read options
ierr = PetscOptionsGetInt(NULL, "-nx", &(data.nx), NULL); CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL, "-ny", &(data.ny), NULL); CHKERRQ(ierr);
hline();
ierr = PetscPrintf(PETSC_COMM_WORLD, "Program that reminds you why you need to use AOApplicationToPetsc if you want to map from P to UV"); CHKERRQ(ierr);
hline();
createArrays(&data);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nCreate matrix G that maps from P to U [VecGetOwnershipRange, MatCreateAIJ]"); CHKERRQ(ierr); hline();
// ownership range of u
ierr = VecGetOwnershipRange(data.uPacked, &uStart, &uEnd); CHKERRQ(ierr);
uLocalSize = uEnd-uStart;
// create arrays to store nnz values
ierr = PetscMalloc(uLocalSize*sizeof(PetscInt), &d_nnz); CHKERRQ(ierr);
ierr = PetscMalloc(uLocalSize*sizeof(PetscInt), &o_nnz); CHKERRQ(ierr);
// ownership range of phi
ierr = VecGetOwnershipRange(data.pGlobal, &pStart, &pEnd); CHKERRQ(ierr);
pLocalSize = pEnd-pStart;
ierr = DMDAGetCorners(data.pda, NULL, NULL, NULL, &pm, &pn, NULL); CHKERRQ(ierr);
// count the number of non-zeros in each row
ierr = DMDAGetAO(data.pda, &pao); CHKERRQ(ierr);
ierr = DMDAGetCorners(data.uda, &umstart, &unstart, NULL, &um, &un, NULL); CHKERRQ(ierr);
localIdx = 0;
for(j=unstart; j<unstart+un; j++)
{
for(i=umstart; i<umstart+um; i++)
{
d_nnz[localIdx] = 0;
o_nnz[localIdx] = 0;
// G portion
col = j*data.nx+i;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
col = pStart + (j-unstart)*pm + i-umstart;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\t", col); CHKERRQ(ierr);
(col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++;
col = j*data.nx+i+1;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
col = pStart + (j-unstart)*pm + i+1-umstart;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\n", col); CHKERRQ(ierr);
(col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++;
localIdx++;
}
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "-\n"); CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr);
ierr = DMDAGetCorners(data.vda, &vmstart, &vnstart, NULL, &vm, &vn, NULL); CHKERRQ(ierr);
for(j=vnstart; j<vnstart+vn; j++)
{
for(i=vmstart; i<vmstart+vm; i++)
{
d_nnz[localIdx] = 0;
o_nnz[localIdx] = 0;
// G portion
col = j*data.nx+i;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
col = pStart + (j-unstart)*pm + i-umstart;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\t", col); CHKERRQ(ierr);
(col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++;
col = (j+1)*data.nx+i;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr);
col = pStart + (j+1-unstart)*pm + i-umstart;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\n", col); CHKERRQ(ierr);
(col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++;
localIdx++;
}
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "-\n"); CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr);
//ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%d + %d = %d\n", m*n, p*q, uLocalSize); CHKERRQ(ierr);
//ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr);
/*
ierr = MatCreateAIJ(PETSC_COMM_WORLD, uLocalSize, pLocalSize, PETSC_DETERMINE, PETSC_DETERMINE, 2, NULL, 1, NULL, &G); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nPopulate G [DMCompositeGetGlobalISs, ISGetIndices, DMDAGetCorners, AOApplicationToPetsc, MatSetValues, MatAssemblyBegin, MatAssemblyEnd]"); CHKERRQ(ierr); hline();
ierr = DMCompositeGetGlobalISs(pack, &is); CHKERRQ(ierr);
ierr = ISGetIndices(is[0], &xindices); CHKERRQ(ierr);
ierr = DMDAGetCorners(uda, &mstart, &nstart, NULL, &m, &n, NULL);
localIdx = 0;
for(j=nstart; j<nstart+n; j++)
{
for(i=mstart; i<mstart+m; i++)
{
row = xindices[localIdx]; // uses the indices obtained using DMCompositeGetGlobalISs and ISGetIndices
cols[0] = j*nx+i+1; // i and j are obtained from DMDAGetCorners
cols[1] = j*nx+i;
ierr = AOApplicationToPetsc(pao, 2, cols); CHKERRQ(ierr); // maps the natural ordering to petsc ordering using the AO obtained from DMDAGetAO
values[0] = 1;
values[1] = -1;
ierr = MatSetValues(G, 1, &row, 2, cols, values, INSERT_VALUES); CHKERRQ(ierr);
localIdx++;
}
}
ierr = ISRestoreIndices(is[0], &xindices); CHKERRQ(ierr);
ierr = ISGetIndices(is[1], &yindices); CHKERRQ(ierr);
ierr = DMDAGetCorners(vda, &mstart, &nstart, NULL, &m, &n, NULL);
localIdx = 0;
for(j=nstart; j<nstart+n; j++)
{
for(i=mstart; i<mstart+m; i++)
{
row = yindices[localIdx];
cols[0] = (j+1)*nx+i;
cols[1] = j*nx+i;
ierr = AOApplicationToPetsc(pao, 2, cols); CHKERRQ(ierr);
values[0] = 1;
values[1] = -1;
ierr = MatSetValues(G, 1, &row, 2, cols, values, INSERT_VALUES); CHKERRQ(ierr);
localIdx++;
}
}
ierr = ISRestoreIndices(is[1], &yindices); CHKERRQ(ierr);
ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatView(G, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nU = G*P [MatMult]"); CHKERRQ(ierr); hline();
ierr = MatMult(G, pGlobal, uPacked); CHKERRQ(ierr);
ierr = VecView(uPacked, PETSC_VIEWER_STDOUT_WORLD);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nC = GT*G [MatTransposeMatMult]"); CHKERRQ(ierr); hline();
ierr = MatTransposeMatMult(G, G, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &C); CHKERRQ(ierr);
ierr = MatView(C, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "\nbc2 = C*P [MatMult]"); CHKERRQ(ierr); hline();
ierr = MatMult(C, pGlobal, bc2); CHKERRQ(ierr);
ierr = VecView(bc2, PETSC_VIEWER_STDOUT_WORLD);
ierr = MatDestroy(&C); CHKERRQ(ierr);
ierr = MatDestroy(&G); CHKERRQ(ierr);
ierr = ISDestroy(&is[0]); CHKERRQ(ierr);
ierr = ISDestroy(&is[1]); CHKERRQ(ierr);
ierr = PetscFree(is); CHKERRQ(ierr);
ierr = VecDestroy(&bc2); CHKERRQ(ierr);
ierr = VecDestroy(&uPacked); CHKERRQ(ierr);
ierr = VecDestroy(&pGlobal); CHKERRQ(ierr);
ierr = VecDestroy(&bc2); CHKERRQ(ierr);
ierr = DMDestroy(&vda); CHKERRQ(ierr);
ierr = DMDestroy(&uda); CHKERRQ(ierr);
ierr = DMDestroy(&pda); CHKERRQ(ierr);
ierr = DMDestroy(&pack); CHKERRQ(ierr);
*/
destroyArrays(&data);
ierr = PetscFinalize();
return 0;
}
PETSC_EXTERN void PETSC_STDCALL dmdagetao_(DM da,AO *ao, int *__ierr ){
*__ierr = DMDAGetAO(
(DM)PetscToPointer((da) ),ao);
}
