/* Given a DMDA generates a VecScatter context that will deliver a slice of the global vector to each processor. In this example, each processor receives the values i=*, j=*, k=rank, i.e. one z plane. Note: This code is written assuming only one degree of freedom per node. For multiple degrees of freedom per node use ISCreateBlock() instead of ISCreateGeneral(). */ PetscErrorCode GenerateSliceScatter(DM da,VecScatter *scatter,Vec *vslice) { AO ao; PetscInt M,N,P,nslice,*sliceindices,count,i,j; PetscMPIInt rank; PetscErrorCode ierr; MPI_Comm comm; Vec vglobal; IS isfrom,isto; ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr); ierr = DMDAGetInfo(da,0,&M,&N,&P,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr); /* nslice is number of degrees of freedom in this processors slice if there are more processors then z plans the extra processors get 0 elements in their slice. */ if (rank < P) nslice = M*N; else nslice = 0; /* Generate the local vector to hold this processors slice */ ierr = VecCreateSeq(PETSC_COMM_SELF,nslice,vslice);CHKERRQ(ierr); ierr = DMCreateGlobalVector(da,&vglobal);CHKERRQ(ierr); /* Generate the indices for the slice in the "natural" global ordering Note: this is just an example, one could select any subset of nodes on each processor. Just list them in the global natural ordering. */ ierr = PetscMalloc1((nslice+1),&sliceindices);CHKERRQ(ierr); count = 0; if (rank < P) { for (j=0; j<N; j++) { for (i=0; i<M; i++) { sliceindices[count++] = rank*M*N + j*M + i; } } } /* Convert the indices to the "PETSc" global ordering */ ierr = AOApplicationToPetsc(ao,nslice,sliceindices);CHKERRQ(ierr); /* Create the "from" and "to" index set */ /* This is to scatter from the global vector */ ierr = ISCreateGeneral(PETSC_COMM_SELF,nslice,sliceindices,PETSC_OWN_POINTER,&isfrom);CHKERRQ(ierr); /* This is to gather into the local vector */ ierr = ISCreateStride(PETSC_COMM_SELF,nslice,0,1,&isto);CHKERRQ(ierr); ierr = VecScatterCreate(vglobal,isfrom,*vslice,isto,scatter);CHKERRQ(ierr); ierr = ISDestroy(&isfrom);CHKERRQ(ierr); ierr = ISDestroy(&isto);CHKERRQ(ierr); return 0; }
/*@ DMDANaturalAllToGlobalCreate - Creates a scatter context that maps from a copy of the entire vector on each processor to its local part in the global vector. Collective on DMDA Input Parameter: . da - the distributed array context Output Parameter: . scatter - the scatter context Level: advanced .keywords: distributed array, global to local, begin, coarse problem .seealso: DMDAGlobalToNaturalEnd(), DMLocalToGlobalBegin(), DMDACreate2d(), DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector() @*/ PetscErrorCode DMDANaturalAllToGlobalCreate(DM da,VecScatter *scatter) { PetscErrorCode ierr; DM_DA *dd = (DM_DA*)da->data; PetscInt M,m = dd->Nlocal,start; IS from,to; Vec tmplocal,global; AO ao; PetscFunctionBegin; PetscValidHeaderSpecific(da,DM_CLASSID,1); PetscValidPointer(scatter,2); ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr); /* create the scatter context */ ierr = MPI_Allreduce(&m,&M,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)da));CHKERRQ(ierr); ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)da),dd->w,m,PETSC_DETERMINE,0,&global);CHKERRQ(ierr); ierr = VecGetOwnershipRange(global,&start,NULL);CHKERRQ(ierr); ierr = ISCreateStride(PetscObjectComm((PetscObject)da),m,start,1,&from);CHKERRQ(ierr); ierr = AOPetscToApplicationIS(ao,from);CHKERRQ(ierr); ierr = ISCreateStride(PetscObjectComm((PetscObject)da),m,start,1,&to);CHKERRQ(ierr); ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,dd->w,M,0,&tmplocal);CHKERRQ(ierr); ierr = VecScatterCreate(tmplocal,from,global,to,scatter);CHKERRQ(ierr); ierr = VecDestroy(&tmplocal);CHKERRQ(ierr); ierr = VecDestroy(&global);CHKERRQ(ierr); ierr = ISDestroy(&from);CHKERRQ(ierr); ierr = ISDestroy(&to);CHKERRQ(ierr); PetscFunctionReturn(0); }
/*@ DMDAGlobalToNaturalAllCreate - Creates a scatter context that maps from the global vector the entire vector to each processor in natural numbering Collective on DMDA Input Parameter: . da - the distributed array context Output Parameter: . scatter - the scatter context Level: advanced .keywords: distributed array, global to local, begin, coarse problem .seealso: DMDAGlobalToNaturalEnd(), DMLocalToGlobalBegin(), DMDACreate2d(), DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector() @*/ PetscErrorCode DMDAGlobalToNaturalAllCreate(DM da,VecScatter *scatter) { PetscErrorCode ierr; PetscInt N; IS from,to; Vec tmplocal,global; AO ao; DM_DA *dd = (DM_DA*)da->data; PetscFunctionBegin; PetscValidHeaderSpecific(da,DM_CLASSID,1); PetscValidPointer(scatter,2); ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr); /* create the scatter context */ ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)da),dd->w,dd->Nlocal,PETSC_DETERMINE,0,&global);CHKERRQ(ierr); ierr = VecGetSize(global,&N);CHKERRQ(ierr); ierr = ISCreateStride(PetscObjectComm((PetscObject)da),N,0,1,&to);CHKERRQ(ierr); ierr = AOPetscToApplicationIS(ao,to);CHKERRQ(ierr); ierr = ISCreateStride(PetscObjectComm((PetscObject)da),N,0,1,&from);CHKERRQ(ierr); ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,dd->w,N,0,&tmplocal);CHKERRQ(ierr); ierr = VecScatterCreate(global,from,tmplocal,to,scatter);CHKERRQ(ierr); ierr = VecDestroy(&tmplocal);CHKERRQ(ierr); ierr = VecDestroy(&global);CHKERRQ(ierr); ierr = ISDestroy(&from);CHKERRQ(ierr); ierr = ISDestroy(&to);CHKERRQ(ierr); PetscFunctionReturn(0); }
/*@C DMDAGetRay - Returns a vector on process zero that contains a row or column of the values in a DMDA vector Collective on DMDA Input Parameters: + da - the distributed array . vec - the vector . dir - Cartesian direction, either DMDA_X, DMDA_Y, or DMDA_Z - gp - global grid point number in this direction Output Parameters: + newvec - the new vector that can hold the values (size zero on all processes except process 0) - scatter - the VecScatter that will map from the original vector to the slice Level: advanced Notes: All processors that share the DMDA must call this with the same gp value .keywords: distributed array, get, processor subset @*/ PetscErrorCode DMDAGetRay(DM da,DMDADirection dir,PetscInt gp,Vec *newvec,VecScatter *scatter) { PetscMPIInt rank; DM_DA *dd = (DM_DA*)da->data; PetscErrorCode ierr; IS is; AO ao; Vec vec; PetscInt *indices,i,j; PetscFunctionBegin; if (dd->dim == 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Cannot get slice from 1d DMDA"); if (dd->dim == 3) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Cannot get slice from 3d DMDA"); ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr); ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)da),&rank);CHKERRQ(ierr); if (!rank) { if (dir == DMDA_Y) { ierr = PetscMalloc(dd->w*dd->M*sizeof(PetscInt),&indices);CHKERRQ(ierr); indices[0] = gp*dd->M*dd->w; for (i=1; i<dd->M*dd->w; i++) indices[i] = indices[i-1] + 1; ierr = AOApplicationToPetsc(ao,dd->M*dd->w,indices);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_SELF,newvec);CHKERRQ(ierr); ierr = VecSetBlockSize(*newvec,dd->w);CHKERRQ(ierr); ierr = VecSetSizes(*newvec,dd->M*dd->w,PETSC_DETERMINE);CHKERRQ(ierr); ierr = VecSetType(*newvec,VECSEQ);CHKERRQ(ierr); ierr = ISCreateGeneral(PETSC_COMM_SELF,dd->w*dd->M,indices,PETSC_OWN_POINTER,&is);CHKERRQ(ierr); } else if (dir == DMDA_X) { ierr = PetscMalloc(dd->w*dd->N*sizeof(PetscInt),&indices);CHKERRQ(ierr); indices[0] = dd->w*gp; for (j=1; j<dd->w; j++) indices[j] = indices[j-1] + 1; for (i=1; i<dd->N; i++) { indices[i*dd->w] = indices[i*dd->w-1] + dd->w*dd->M - dd->w + 1; for (j=1; j<dd->w; j++) indices[i*dd->w + j] = indices[i*dd->w + j - 1] + 1; } ierr = AOApplicationToPetsc(ao,dd->w*dd->N,indices);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_SELF,newvec);CHKERRQ(ierr); ierr = VecSetBlockSize(*newvec,dd->w);CHKERRQ(ierr); ierr = VecSetSizes(*newvec,dd->N*dd->w,PETSC_DETERMINE);CHKERRQ(ierr); ierr = VecSetType(*newvec,VECSEQ);CHKERRQ(ierr); ierr = ISCreateGeneral(PETSC_COMM_SELF,dd->w*dd->N,indices,PETSC_OWN_POINTER,&is);CHKERRQ(ierr); } else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Unknown DMDADirection"); } else { ierr = VecCreateSeq(PETSC_COMM_SELF,0,newvec);CHKERRQ(ierr); ierr = ISCreateGeneral(PETSC_COMM_SELF,0,0,PETSC_COPY_VALUES,&is);CHKERRQ(ierr); } ierr = DMGetGlobalVector(da,&vec);CHKERRQ(ierr); ierr = VecScatterCreate(vec,is,*newvec,NULL,scatter);CHKERRQ(ierr); ierr = DMRestoreGlobalVector(da,&vec);CHKERRQ(ierr); ierr = ISDestroy(&is);CHKERRQ(ierr); PetscFunctionReturn(0); }
static PetscErrorCode ComputeMapping(DomainData dd,ISLocalToGlobalMapping *isg2lmap) { PetscErrorCode ierr; DM da; AO ao; DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE, bz = DM_BOUNDARY_NONE; DMDAStencilType stype = DMDA_STENCIL_BOX; ISLocalToGlobalMapping temp_isg2lmap; PetscInt i,j,k,ig,jg,kg,lindex,gindex,localsize; PetscInt *global_indices; PetscFunctionBeginUser; /* Not an efficient mapping: this function computes a very simple lexicographic mapping just to illustrate the creation of a MATIS object */ localsize = dd.xm_l*dd.ym_l*dd.zm_l; ierr = PetscMalloc1(localsize,&global_indices);CHKERRQ(ierr); for (k=0; k<dd.zm_l; k++) { kg=dd.startz+k; for (j=0; j<dd.ym_l; j++) { jg=dd.starty+j; for (i=0; i<dd.xm_l; i++) { ig =dd.startx+i; lindex =k*dd.xm_l*dd.ym_l+j*dd.xm_l+i; gindex =kg*dd.xm*dd.ym+jg*dd.xm+ig; global_indices[lindex]=gindex; } } } if (dd.dim==3) { ierr = DMDACreate3d(dd.gcomm,bx,by,bz,stype,dd.xm,dd.ym,dd.zm,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,NULL,&da);CHKERRQ(ierr); } else if (dd.dim==2) { ierr = DMDACreate2d(dd.gcomm,bx,by,stype,dd.xm,dd.ym,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr); } else { ierr = DMDACreate1d(dd.gcomm,bx,dd.xm,1,1,NULL,&da);CHKERRQ(ierr); } ierr = DMDASetAOType(da,AOMEMORYSCALABLE);CHKERRQ(ierr); ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr); ierr = AOApplicationToPetsc(ao,dd.xm_l*dd.ym_l*dd.zm_l,global_indices);CHKERRQ(ierr); ierr = ISLocalToGlobalMappingCreate(dd.gcomm,1,localsize,global_indices,PETSC_OWN_POINTER,&temp_isg2lmap);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); *isg2lmap = temp_isg2lmap; PetscFunctionReturn(0); }
int main(int argc,char **argv) { PetscMPIInt rank; PetscErrorCode ierr; PetscInt M = 10,N = 8,m = PETSC_DECIDE; PetscInt s =2,w=2,n = PETSC_DECIDE,nloc,l,i,j,kk; PetscInt Xs,Xm,Ys,Ym,iloc,*iglobal; const PetscInt *ltog; PetscInt *lx = NULL,*ly = NULL; PetscBool testorder = PETSC_FALSE,flg; DMBoundaryType bx = DM_BOUNDARY_NONE,by= DM_BOUNDARY_NONE; DM da; PetscViewer viewer; Vec local,global; PetscScalar value; DMDAStencilType st = DMDA_STENCIL_BOX; AO ao; ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,400,&viewer);CHKERRQ(ierr); /* Readoptions */ ierr = PetscOptionsGetInt(NULL,NULL,"-NX",&M,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-NY",&N,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-s",&s,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-w",&w,NULL);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-xperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_PERIODIC; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-yperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_PERIODIC; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-xghosted",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_GHOSTED; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-yghosted",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_GHOSTED; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-star",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_STAR; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-box",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_BOX; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-testorder",&testorder,NULL);CHKERRQ(ierr); /* Test putting two nodes in x and y on each processor, exact last processor in x and y gets the rest. */ flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-distribute",&flg,NULL);CHKERRQ(ierr); if (flg) { if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option"); ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr); for (i=0; i<m-1; i++) { lx[i] = 4;} lx[m-1] = M - 4*(m-1); if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option"); ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr); for (i=0; i<n-1; i++) { ly[i] = 2;} ly[n-1] = N - 2*(n-1); } /* Create distributed array and get vectors */ ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,st,M,N,m,n,w,s,lx,ly,&da);CHKERRQ(ierr); ierr = PetscFree(lx);CHKERRQ(ierr); ierr = PetscFree(ly);CHKERRQ(ierr); ierr = DMView(da,viewer);CHKERRQ(ierr); ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr); /* Set global vector; send ghost points to local vectors */ value = 1; ierr = VecSet(global,value);CHKERRQ(ierr); ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr); /* Scale local vectors according to processor rank; pass to global vector */ ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); value = rank; ierr = VecScale(local,value);CHKERRQ(ierr); ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr); ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr); if (!testorder) { /* turn off printing when testing ordering mappings */ ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vectors:\n");CHKERRQ(ierr); ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n");CHKERRQ(ierr); } /* Send ghost points to local vectors */ ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-local_print",&flg,NULL);CHKERRQ(ierr); if (flg) { PetscViewer sviewer; ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr); ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr); ierr = VecView(local,sviewer);CHKERRQ(ierr); ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr); ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Tests mappings betweeen application/PETSc orderings */ if (testorder) { ISLocalToGlobalMapping ltogm; ierr = DMGetLocalToGlobalMapping(da,<ogm);CHKERRQ(ierr); ierr = ISLocalToGlobalMappingGetSize(ltogm,&nloc);CHKERRQ(ierr); ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr); ierr = DMDAGetGhostCorners(da,&Xs,&Ys,NULL,&Xm,&Ym,NULL);CHKERRQ(ierr); ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr); ierr = PetscMalloc1(nloc,&iglobal);CHKERRQ(ierr); /* Set iglobal to be global indices for each processor's local and ghost nodes, using the DMDA ordering of grid points */ kk = 0; for (j=Ys; j<Ys+Ym; j++) { for (i=Xs; i<Xs+Xm; i++) { iloc = w*((j-Ys)*Xm + i-Xs); for (l=0; l<w; l++) { iglobal[kk++] = ltog[iloc+l]; } } } /* Map this to the application ordering (which for DMDAs is just the natural ordering that would be used for 1 processor, numbering most rapidly by x, then y) */ ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr); /* Then map the application ordering back to the PETSc DMDA ordering */ ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr); /* Verify the mappings */ kk=0; for (j=Ys; j<Ys+Ym; j++) { for (i=Xs; i<Xs+Xm; i++) { iloc = w*((j-Ys)*Xm + i-Xs); for (l=0; l<w; l++) { if (iglobal[kk] != ltog[iloc+l]) { ierr = PetscFPrintf(PETSC_COMM_SELF,stdout,"[%d] Problem with mapping: j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",rank,j,i,l,ltog[iloc+l],iglobal[kk]);CHKERRQ(ierr); } kk++; } } } ierr = PetscFree(iglobal);CHKERRQ(ierr); ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr); } /* Free memory */ ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); ierr = VecDestroy(&local);CHKERRQ(ierr); ierr = VecDestroy(&global);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
int main(int argc,char **argv) { PetscMPIInt rank; PetscInt M = 3,N = 5,P=3,s=1,w=2,nloc,l,i,j,k,kk,m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE; PetscErrorCode ierr; PetscInt Xs,Xm,Ys,Ym,Zs,Zm,iloc,*ltog,*iglobal; PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL,*lz = PETSC_NULL; PetscBool test_order = PETSC_FALSE; DM da; PetscViewer viewer; Vec local,global; PetscScalar value; DMDABoundaryType bx = DMDA_BOUNDARY_NONE,by = DMDA_BOUNDARY_NONE,bz = DMDA_BOUNDARY_NONE; DMDAStencilType stencil_type = DMDA_STENCIL_BOX; AO ao; PetscBool flg = PETSC_FALSE; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,300,&viewer);CHKERRQ(ierr); /* Read options */ ierr = PetscOptionsGetInt(PETSC_NULL,"-NX",&M,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-NY",&N,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-NZ",&P,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-w",&w,PETSC_NULL);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) stencil_type = DMDA_STENCIL_STAR; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-box",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) stencil_type = DMDA_STENCIL_BOX; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-xperiodic",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) bx = DMDA_BOUNDARY_PERIODIC; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-xghosted",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) bx = DMDA_BOUNDARY_GHOSTED; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-xnonghosted",&flg,PETSC_NULL);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-yperiodic",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) by = DMDA_BOUNDARY_PERIODIC; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-yghosted",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) by = DMDA_BOUNDARY_GHOSTED; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-ynonghosted",&flg,PETSC_NULL);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-zperiodic",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) bz = DMDA_BOUNDARY_PERIODIC; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-zghosted",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) bz = DMDA_BOUNDARY_GHOSTED; flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-znonghosted",&flg,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetBool(PETSC_NULL,"-testorder",&test_order,PETSC_NULL);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-distribute",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) { if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option"); ierr = PetscMalloc(m*sizeof(PetscInt),&lx);CHKERRQ(ierr); for (i=0; i<m-1; i++) { lx[i] = 4;} lx[m-1] = M - 4*(m-1); if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option"); ierr = PetscMalloc(n*sizeof(PetscInt),&ly);CHKERRQ(ierr); for (i=0; i<n-1; i++) { ly[i] = 2;} ly[n-1] = N - 2*(n-1); if (p == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -p option with -distribute option"); ierr = PetscMalloc(p*sizeof(PetscInt),&lz);CHKERRQ(ierr); for (i=0; i<p-1; i++) { lz[i] = 2;} lz[p-1] = P - 2*(p-1); } /* Create distributed array and get vectors */ ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,m,n,p,w,s,lx,ly,lz,&da);CHKERRQ(ierr); ierr = PetscFree(lx);CHKERRQ(ierr); ierr = PetscFree(ly);CHKERRQ(ierr); ierr = PetscFree(lz);CHKERRQ(ierr); ierr = DMView(da,viewer);CHKERRQ(ierr); ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr); /* Set global vector; send ghost points to local vectors */ value = 1; ierr = VecSet(global,value);CHKERRQ(ierr); ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr); /* Scale local vectors according to processor rank; pass to global vector */ ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); value = rank; ierr = VecScale(local,value);CHKERRQ(ierr); ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr); ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr); if (!test_order) { /* turn off printing when testing ordering mappings */ if (M*N*P<40) { ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vector:\n");CHKERRQ(ierr); ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr); } } /* Send ghost points to local vectors */ ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-local_print",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) { PetscViewer sviewer; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr); ierr = PetscViewerGetSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr); ierr = VecView(local,sviewer);CHKERRQ(ierr); ierr = PetscViewerRestoreSingleton(PETSC_VIEWER_STDOUT_WORLD,&sviewer);CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr); } /* Tests mappings betweeen application/PETSc orderings */ if (test_order) { ierr = DMDAGetGhostCorners(da,&Xs,&Ys,&Zs,&Xm,&Ym,&Zm);CHKERRQ(ierr); ierr = DMDAGetGlobalIndices(da,&nloc,<og);CHKERRQ(ierr); ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr); /* ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */ ierr = PetscMalloc(nloc*sizeof(PetscInt),&iglobal);CHKERRQ(ierr); /* Set iglobal to be global indices for each processor's local and ghost nodes, using the DMDA ordering of grid points */ kk = 0; for (k=Zs; k<Zs+Zm; k++) { for (j=Ys; j<Ys+Ym; j++) { for (i=Xs; i<Xs+Xm; i++) { iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs); for (l=0; l<w; l++) { iglobal[kk++] = ltog[iloc+l]; } } } } /* Map this to the application ordering (which for DMDAs is just the natural ordering that would be used for 1 processor, numbering most rapidly by x, then y, then z) */ ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr); /* Then map the application ordering back to the PETSc DMDA ordering */ ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr); /* Verify the mappings */ kk=0; for (k=Zs; k<Zs+Zm; k++) { for (j=Ys; j<Ys+Ym; j++) { for (i=Xs; i<Xs+Xm; i++) { iloc = w*((k-Zs)*Xm*Ym + (j-Ys)*Xm + i-Xs); for (l=0; l<w; l++) { if (iglobal[kk] != ltog[iloc+l]) { ierr = PetscPrintf(MPI_COMM_WORLD,"[%D] Problem with mapping: z=%D, j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n", rank,k,j,i,l,ltog[iloc+l],iglobal[kk]); } kk++; } } } } ierr = PetscFree(iglobal);CHKERRQ(ierr); } /* Free memory */ ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); ierr = VecDestroy(&local);CHKERRQ(ierr); ierr = VecDestroy(&global);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
int main(int argc,char **argv) { PetscErrorCode ierr; PetscInt rank; PetscInt i, j; PetscInt um, un, vm, vn, pm, pn; PetscInt umstart, unstart, vmstart, vnstart; PetscInt *d_nnz, *o_nnz; PetscInt uStart, uEnd, pStart, pEnd, uLocalSize, pLocalSize; PetscInt col; //PetscInt row, cols[2]; PetscInt localIdx; AO pao; //PetscScalar values[2]; //IS *is; //const PetscInt *xindices, *yindices; simInfo data; ierr = PetscInitialize(&argc, &argv, NULL, NULL); CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr); // defaults data.nx = 5; data.ny = 5; // Read options ierr = PetscOptionsGetInt(NULL, "-nx", &(data.nx), NULL); CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL, "-ny", &(data.ny), NULL); CHKERRQ(ierr); hline(); ierr = PetscPrintf(PETSC_COMM_WORLD, "Program that reminds you why you need to use AOApplicationToPetsc if you want to map from P to UV"); CHKERRQ(ierr); hline(); createArrays(&data); ierr = PetscPrintf(PETSC_COMM_WORLD, "\nCreate matrix G that maps from P to U [VecGetOwnershipRange, MatCreateAIJ]"); CHKERRQ(ierr); hline(); // ownership range of u ierr = VecGetOwnershipRange(data.uPacked, &uStart, &uEnd); CHKERRQ(ierr); uLocalSize = uEnd-uStart; // create arrays to store nnz values ierr = PetscMalloc(uLocalSize*sizeof(PetscInt), &d_nnz); CHKERRQ(ierr); ierr = PetscMalloc(uLocalSize*sizeof(PetscInt), &o_nnz); CHKERRQ(ierr); // ownership range of phi ierr = VecGetOwnershipRange(data.pGlobal, &pStart, &pEnd); CHKERRQ(ierr); pLocalSize = pEnd-pStart; ierr = DMDAGetCorners(data.pda, NULL, NULL, NULL, &pm, &pn, NULL); CHKERRQ(ierr); // count the number of non-zeros in each row ierr = DMDAGetAO(data.pda, &pao); CHKERRQ(ierr); ierr = DMDAGetCorners(data.uda, &umstart, &unstart, NULL, &um, &un, NULL); CHKERRQ(ierr); localIdx = 0; for(j=unstart; j<unstart+un; j++) { for(i=umstart; i<umstart+um; i++) { d_nnz[localIdx] = 0; o_nnz[localIdx] = 0; // G portion col = j*data.nx+i; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr); ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr); ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr); col = pStart + (j-unstart)*pm + i-umstart; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\t", col); CHKERRQ(ierr); (col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++; col = j*data.nx+i+1; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr); ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr); ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr); col = pStart + (j-unstart)*pm + i+1-umstart; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\n", col); CHKERRQ(ierr); (col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++; localIdx++; } } ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "-\n"); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr); ierr = DMDAGetCorners(data.vda, &vmstart, &vnstart, NULL, &vm, &vn, NULL); CHKERRQ(ierr); for(j=vnstart; j<vnstart+vn; j++) { for(i=vmstart; i<vmstart+vm; i++) { d_nnz[localIdx] = 0; o_nnz[localIdx] = 0; // G portion col = j*data.nx+i; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr); ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr); ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr); col = pStart + (j-unstart)*pm + i-umstart; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\t", col); CHKERRQ(ierr); (col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++; col = (j+1)*data.nx+i; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr); ierr = AOApplicationToPetsc(pao, 1, &col); CHKERRQ(ierr); ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d,", col); CHKERRQ(ierr); col = pStart + (j+1-unstart)*pm + i-umstart; ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%3d\n", col); CHKERRQ(ierr); (col>=pStart && col<pEnd)? d_nnz[localIdx]++ : o_nnz[localIdx]++; localIdx++; } } ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "-\n"); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr); //ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%d + %d = %d\n", m*n, p*q, uLocalSize); CHKERRQ(ierr); //ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD); CHKERRQ(ierr); /* ierr = MatCreateAIJ(PETSC_COMM_WORLD, uLocalSize, pLocalSize, PETSC_DETERMINE, PETSC_DETERMINE, 2, NULL, 1, NULL, &G); CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD, "\nPopulate G [DMCompositeGetGlobalISs, ISGetIndices, DMDAGetCorners, AOApplicationToPetsc, MatSetValues, MatAssemblyBegin, MatAssemblyEnd]"); CHKERRQ(ierr); hline(); ierr = DMCompositeGetGlobalISs(pack, &is); CHKERRQ(ierr); ierr = ISGetIndices(is[0], &xindices); CHKERRQ(ierr); ierr = DMDAGetCorners(uda, &mstart, &nstart, NULL, &m, &n, NULL); localIdx = 0; for(j=nstart; j<nstart+n; j++) { for(i=mstart; i<mstart+m; i++) { row = xindices[localIdx]; // uses the indices obtained using DMCompositeGetGlobalISs and ISGetIndices cols[0] = j*nx+i+1; // i and j are obtained from DMDAGetCorners cols[1] = j*nx+i; ierr = AOApplicationToPetsc(pao, 2, cols); CHKERRQ(ierr); // maps the natural ordering to petsc ordering using the AO obtained from DMDAGetAO values[0] = 1; values[1] = -1; ierr = MatSetValues(G, 1, &row, 2, cols, values, INSERT_VALUES); CHKERRQ(ierr); localIdx++; } } ierr = ISRestoreIndices(is[0], &xindices); CHKERRQ(ierr); ierr = ISGetIndices(is[1], &yindices); CHKERRQ(ierr); ierr = DMDAGetCorners(vda, &mstart, &nstart, NULL, &m, &n, NULL); localIdx = 0; for(j=nstart; j<nstart+n; j++) { for(i=mstart; i<mstart+m; i++) { row = yindices[localIdx]; cols[0] = (j+1)*nx+i; cols[1] = j*nx+i; ierr = AOApplicationToPetsc(pao, 2, cols); CHKERRQ(ierr); values[0] = 1; values[1] = -1; ierr = MatSetValues(G, 1, &row, 2, cols, values, INSERT_VALUES); CHKERRQ(ierr); localIdx++; } } ierr = ISRestoreIndices(is[1], &yindices); CHKERRQ(ierr); ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); ierr = MatView(G, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD, "\nU = G*P [MatMult]"); CHKERRQ(ierr); hline(); ierr = MatMult(G, pGlobal, uPacked); CHKERRQ(ierr); ierr = VecView(uPacked, PETSC_VIEWER_STDOUT_WORLD); ierr = PetscPrintf(PETSC_COMM_WORLD, "\nC = GT*G [MatTransposeMatMult]"); CHKERRQ(ierr); hline(); ierr = MatTransposeMatMult(G, G, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &C); CHKERRQ(ierr); ierr = MatView(C, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD, "\nbc2 = C*P [MatMult]"); CHKERRQ(ierr); hline(); ierr = MatMult(C, pGlobal, bc2); CHKERRQ(ierr); ierr = VecView(bc2, PETSC_VIEWER_STDOUT_WORLD); ierr = MatDestroy(&C); CHKERRQ(ierr); ierr = MatDestroy(&G); CHKERRQ(ierr); ierr = ISDestroy(&is[0]); CHKERRQ(ierr); ierr = ISDestroy(&is[1]); CHKERRQ(ierr); ierr = PetscFree(is); CHKERRQ(ierr); ierr = VecDestroy(&bc2); CHKERRQ(ierr); ierr = VecDestroy(&uPacked); CHKERRQ(ierr); ierr = VecDestroy(&pGlobal); CHKERRQ(ierr); ierr = VecDestroy(&bc2); CHKERRQ(ierr); ierr = DMDestroy(&vda); CHKERRQ(ierr); ierr = DMDestroy(&uda); CHKERRQ(ierr); ierr = DMDestroy(&pda); CHKERRQ(ierr); ierr = DMDestroy(&pack); CHKERRQ(ierr); */ destroyArrays(&data); ierr = PetscFinalize(); return 0; }
PETSC_EXTERN void PETSC_STDCALL dmdagetao_(DM da,AO *ao, int *__ierr ){ *__ierr = DMDAGetAO( (DM)PetscToPointer((da) ),ao); }