int main(int argc,char **argv)
{
PetscErrorCode ierr;
AO ao;
PetscInt *localvert=NULL, nlocal;
PetscMPIInt rank;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscMalloc(4*sizeof(PetscInt),&localvert);CHKERRQ(ierr);
if (!rank) {
nlocal = 4;
localvert[0] = 0;
localvert[1] = 1;
localvert[2] = 2;
localvert[3] = 3;
} else {
nlocal = 0;
}
/* Test AOCreateBasic() */
ierr = AOCreateBasic(PETSC_COMM_WORLD, nlocal, localvert, NULL, &ao);CHKERRQ(ierr);
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* Test AOCreateMemoryScalable() */
ierr = AOCreateMemoryScalable(PETSC_COMM_WORLD, nlocal, localvert, NULL, &ao);CHKERRQ(ierr);
ierr = AODestroy(&ao);CHKERRQ(ierr);
ierr = PetscFree(localvert);CHKERRQ(ierr);
ierr = PetscFinalize();
PetscFunctionReturn(0);
}
PetscErrorCode DMDestroy_DA(DM da)
{
PetscErrorCode ierr;
PetscErrorCode i;
DM_DA *dd = (DM_DA*)da->data;
PetscFunctionBegin;
/* destroy the external/common part */
for (i=0; i<DMDA_MAX_WORK_ARRAYS; i++) {
ierr = PetscFree(dd->startghostedout[i]);CHKERRQ(ierr);
ierr = PetscFree(dd->startghostedin[i]);CHKERRQ(ierr);
ierr = PetscFree(dd->startout[i]);CHKERRQ(ierr);
ierr = PetscFree(dd->startin[i]);CHKERRQ(ierr);
}
ierr = VecScatterDestroy(&dd->gtol);CHKERRQ(ierr);
ierr = VecScatterDestroy(&dd->ltol);CHKERRQ(ierr);
ierr = VecDestroy(&dd->natural);CHKERRQ(ierr);
ierr = VecScatterDestroy(&dd->gton);CHKERRQ(ierr);
ierr = AODestroy(&dd->ao);CHKERRQ(ierr);
ierr = PetscFree(dd->aotype);CHKERRQ(ierr);
ierr = PetscFree(dd->lx);CHKERRQ(ierr);
ierr = PetscFree(dd->ly);CHKERRQ(ierr);
ierr = PetscFree(dd->lz);CHKERRQ(ierr);
if (dd->fieldname) {
for (i=0; i<dd->w; i++) {
ierr = PetscFree(dd->fieldname[i]);CHKERRQ(ierr);
}
ierr = PetscFree(dd->fieldname);CHKERRQ(ierr);
}
if (dd->coordinatename) {
for (i=0; i<da->dim; i++) {
ierr = PetscFree(dd->coordinatename[i]);CHKERRQ(ierr);
}
ierr = PetscFree(dd->coordinatename);CHKERRQ(ierr);
}
ierr = ISColoringDestroy(&dd->localcoloring);CHKERRQ(ierr);
ierr = ISColoringDestroy(&dd->ghostedcoloring);CHKERRQ(ierr);
ierr = PetscFree(dd->neighbors);CHKERRQ(ierr);
ierr = PetscFree(dd->dfill);CHKERRQ(ierr);
ierr = PetscFree(dd->ofill);CHKERRQ(ierr);
ierr = PetscFree(dd->ofillcols);CHKERRQ(ierr);
ierr = PetscFree(dd->e);CHKERRQ(ierr);
/* ierr = PetscSectionDestroy(&dd->defaultGlobalSection);CHKERRQ(ierr); */
/* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
ierr = PetscFree(dd);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInt n,*ispetsc,*isapp,start,N,i;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); n = rank + 2;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/* create the orderings */
ierr = PetscMalloc2(n,&ispetsc,n,&isapp);CHKERRQ(ierr);
ierr = MPI_Scan(&n,&start,1,MPIU_INT,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = MPI_Allreduce(&n,&N,1,MPIU_INT,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
start -= n;
for (i=0; i<n; i++) {
ispetsc[i] = start + i;
isapp[i] = N - start - i - 1;
}
/* create the application ordering */
ierr = AOCreateBasic(PETSC_COMM_WORLD,n,isapp,ispetsc,&ao);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* check the mapping */
ierr = AOPetscToApplication(ao,n,ispetsc);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (ispetsc[i] != isapp[i]) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"[%d] Problem with mapping %D to %D\n",rank,i,ispetsc[i]);
}
}
ierr = PetscFree2(ispetsc,isapp);CHKERRQ(ierr);
ierr = AODestroy(&ao);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void MeshData::destroyPointers(){
if (P_pid()>=1 && rowToImport){
MatDestroy(&joinNodes);
MatDestroy(&updateValues);
}
if (pMS) { delete pMS; pMS = 0; }
if (rowToImport){ delete[] rowToImport; rowToImport = 0; }
if (F_rows) { delete[] F_rows; F_rows=0; }
if (F_cols) { delete[] F_cols; F_cols=0; }
if (pos) { delete[] pos; pos=0; }
if (idxn) { delete[] idxn; idxn=0; }
if (idxFreecols) { delete[] idxFreecols; idxFreecols=0; }
if (FP_Array) { delete[] FP_Array; FP_Array=0; }
if (localIDs) { delete[] localIDs; localIDs=0; }
dirichlet.clear();
setOfDomains.clear();
localIDNumbering.clear();
mapPB_nodes.clear();
AODestroy(&ao);
}
int main(int argc,char **argv)
{
PetscInt n = 5;
PetscErrorCode ierr;
PetscMPIInt rank,size;
IS ispetsc,isapp;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/* create the index sets */
ierr = ISCreateStride(PETSC_COMM_WORLD,n,rank,size,&ispetsc);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_WORLD,n,n*rank,1,&isapp);CHKERRQ(ierr);
/* create the application ordering */
ierr = AOCreateBasicIS(isapp,ispetsc,&ao);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(ispetsc,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(isapp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = AOPetscToApplicationIS(ao,ispetsc);CHKERRQ(ierr);
ierr = ISView(isapp,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(ispetsc,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&ispetsc);CHKERRQ(ierr);
ierr = ISDestroy(&isapp);CHKERRQ(ierr);
ierr = AODestroy(&ao);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@
DADestroy - Destroys a distributed array.
Collective on DA
Input Parameter:
. da - the distributed array to destroy
Level: beginner
.keywords: distributed array, destroy
.seealso: DACreate1d(), DACreate2d(), DACreate3d()
@*/
PetscErrorCode PETSCDM_DLLEXPORT DADestroy(DA da)
{
PetscErrorCode ierr;
PetscErrorCode i;
PetscTruth done;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_COOKIE,1);
ierr = DMDestroy_Private((DM)da,&done);CHKERRQ(ierr);
if (!done) PetscFunctionReturn(0);
/* destroy the internal part */
if (da->ops->destroy) {
ierr = (*da->ops->destroy)(da);CHKERRQ(ierr);
}
/* destroy the external/common part */
for (i=0; i<DA_MAX_AD_ARRAYS; i++) {
ierr = PetscFree(da->adstartghostedout[i]);CHKERRQ(ierr);
ierr = PetscFree(da->adstartghostedin[i]);CHKERRQ(ierr);
ierr = PetscFree(da->adstartout[i]);CHKERRQ(ierr);
ierr = PetscFree(da->adstartin[i]);CHKERRQ(ierr);
}
for (i=0; i<DA_MAX_AD_ARRAYS; i++) {
ierr = PetscFree(da->admfstartghostedout[i]);CHKERRQ(ierr);
ierr = PetscFree(da->admfstartghostedin[i]);CHKERRQ(ierr);
ierr = PetscFree(da->admfstartout[i]);CHKERRQ(ierr);
ierr = PetscFree(da->admfstartin[i]);CHKERRQ(ierr);
}
for (i=0; i<DA_MAX_WORK_ARRAYS; i++) {
ierr = PetscFree(da->startghostedout[i]);CHKERRQ(ierr);
ierr = PetscFree(da->startghostedin[i]);CHKERRQ(ierr);
ierr = PetscFree(da->startout[i]);CHKERRQ(ierr);
ierr = PetscFree(da->startin[i]);CHKERRQ(ierr);
}
/* if memory was published with AMS then destroy it */
ierr = PetscObjectDepublish(da);CHKERRQ(ierr);
if (da->ltog) {ierr = VecScatterDestroy(da->ltog);CHKERRQ(ierr);}
if (da->gtol) {ierr = VecScatterDestroy(da->gtol);CHKERRQ(ierr);}
if (da->ltol) {ierr = VecScatterDestroy(da->ltol);CHKERRQ(ierr);}
if (da->natural){
ierr = VecDestroy(da->natural);CHKERRQ(ierr);
}
if (da->gton) {
ierr = VecScatterDestroy(da->gton);CHKERRQ(ierr);
}
if (da->ao) {
ierr = AODestroy(da->ao);CHKERRQ(ierr);
}
if (da->ltogmap) {
ierr = ISLocalToGlobalMappingDestroy(da->ltogmap);CHKERRQ(ierr);
}
if (da->ltogmapb) {
ierr = ISLocalToGlobalMappingDestroy(da->ltogmapb);CHKERRQ(ierr);
}
ierr = PetscFree(da->lx);CHKERRQ(ierr);
ierr = PetscFree(da->ly);CHKERRQ(ierr);
ierr = PetscFree(da->lz);CHKERRQ(ierr);
if (da->fieldname) {
for (i=0; i<da->w; i++) {
ierr = PetscStrfree(da->fieldname[i]);CHKERRQ(ierr);
}
ierr = PetscFree(da->fieldname);CHKERRQ(ierr);
}
if (da->localcoloring) {
ierr = ISColoringDestroy(da->localcoloring);CHKERRQ(ierr);
}
if (da->ghostedcoloring) {
ierr = ISColoringDestroy(da->ghostedcoloring);CHKERRQ(ierr);
}
if (da->coordinates) {ierr = VecDestroy(da->coordinates);CHKERRQ(ierr);}
if (da->ghosted_coordinates) {ierr = VecDestroy(da->ghosted_coordinates);CHKERRQ(ierr);}
if (da->da_coordinates && da != da->da_coordinates) {ierr = DADestroy(da->da_coordinates);CHKERRQ(ierr);}
ierr = PetscFree(da->neighbors);CHKERRQ(ierr);
ierr = PetscFree(da->dfill);CHKERRQ(ierr);
ierr = PetscFree(da->ofill);CHKERRQ(ierr);
ierr = PetscFree(da->e);CHKERRQ(ierr);
ierr = PetscHeaderDestroy(da);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt i,n = 5;
PetscInt getpetsc[] = {0,3,4},getapp[] = {2,1,9,7};
PetscInt getpetsc1[] = {0,3,4},getapp1[] = {2,1,9,7};
PetscInt getpetsc2[] = {0,3,4},getapp2[] = {2,1,9,7};
PetscInt getpetsc3[] = {0,3,4},getapp3[] = {2,1,9,7};
PetscInt getpetsc4[] = {0,3,4},getapp4[] = {2,1,9,7};
PetscMPIInt rank,size;
IS ispetsc,isapp;
AO ao;
const PetscInt *app;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/* create the index sets */
ierr = ISCreateStride(PETSC_COMM_WORLD,n,rank,size,&isapp);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_WORLD,n,n*rank,1,&ispetsc);CHKERRQ(ierr); /* natural numbering */
/* create the application ordering */
ierr = AOCreateBasicIS(isapp,ispetsc,&ao);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] 2,1,9,7 PetscToApplication %D %D %D %D\n",rank,getapp[0],getapp[1],getapp[2],getapp[3]);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] 0,3,4 ApplicationToPetsc %D %D %D\n",rank,getpetsc[0],getpetsc[1],getpetsc[2]);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* test MemoryScalable ao */
/*-------------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTest AOCreateMemoryScalable: \n");CHKERRQ(ierr);
ierr = AOCreateMemoryScalableIS(isapp,ispetsc,&ao);CHKERRQ(ierr);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp1);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc1);CHKERRQ(ierr);
/* Check accuracy */;
for (i=0; i<4; i++) {
if (getapp1[i] != getapp[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getapp1 %d != getapp %d",getapp1[i],getapp[i]);
}
for (i=0; i<3; i++) {
if (getpetsc1[i] != getpetsc[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getpetsc1 %d != getpetsc %d",getpetsc1[i],getpetsc[i]);
}
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* test MemoryScalable ao: ispetsc = NULL */
/*-----------------------------------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTest AOCreateMemoryScalable with ispetsc=NULL:\n");CHKERRQ(ierr);
ierr = AOCreateMemoryScalableIS(isapp,NULL,&ao);CHKERRQ(ierr);CHKERRQ(ierr);
ierr = AOView(ao,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp2);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc2);CHKERRQ(ierr);
/* Check accuracy */;
for (i=0; i<4; i++) {
if (getapp2[i] != getapp[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getapp2 %d != getapp %d",getapp2[i],getapp[i]);
}
for (i=0; i<3; i++) {
if (getpetsc2[i] != getpetsc[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getpetsc2 %d != getpetsc %d",getpetsc2[i],getpetsc[i]);
}
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* test AOCreateMemoryScalable() ao: */
ierr = ISGetIndices(isapp,&app);CHKERRQ(ierr);
ierr = AOCreateMemoryScalable(PETSC_COMM_WORLD,n,app,NULL,&ao);CHKERRQ(ierr);
ierr = ISRestoreIndices(isapp,&app);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp4);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc4);CHKERRQ(ierr);
/* Check accuracy */;
for (i=0; i<4; i++) {
if (getapp4[i] != getapp[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getapp4 %d != getapp %d",getapp4[i],getapp[i]);
}
for (i=0; i<3; i++) {
if (getpetsc4[i] != getpetsc[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getpetsc4 %d != getpetsc %d",getpetsc4[i],getpetsc[i]);
}
ierr = AODestroy(&ao);CHKERRQ(ierr);
/* test general API */
/*------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nTest general API: \n");CHKERRQ(ierr);
ierr = AOCreate(PETSC_COMM_WORLD,&ao);CHKERRQ(ierr);
ierr = AOSetIS(ao,isapp,ispetsc);CHKERRQ(ierr);
ierr = AOSetType(ao,AOMEMORYSCALABLE);CHKERRQ(ierr);
ierr = AOSetFromOptions(ao);CHKERRQ(ierr);
/* ispetsc and isapp are nolonger used. */
ierr = ISDestroy(&ispetsc);CHKERRQ(ierr);
ierr = ISDestroy(&isapp);CHKERRQ(ierr);
ierr = AOPetscToApplication(ao,4,getapp3);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,3,getpetsc3);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] 2,1,9,7 PetscToApplication %D %D %D %D\n",rank,getapp3[0],getapp3[1],getapp3[2],getapp3[3]);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] 0,3,4 ApplicationToPetsc %D %D %D\n",rank,getpetsc3[0],getpetsc3[1],getpetsc3[2]);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
/* Check accuracy */;
for (i=0; i<4; i++) {
if (getapp3[i] != getapp[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getapp3 %d != getapp %d",getapp3[i],getapp[i]);
}
for (i=0; i<3; i++) {
if (getpetsc3[i] != getpetsc[i]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_USER,"getpetsc3 %d != getpetsc %d",getpetsc3[i],getpetsc[i]);
}
ierr = AODestroy(&ao);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode FACreate(FA *infa)
{
FA fa;
PetscMPIInt rank;
PetscInt tonglobal,globalrstart,x,nx,y,ny,*tonatural,i,j,*to,*from,offt[3];
PetscInt *fromnatural,fromnglobal,nscat,nlocal,cntl1,cntl2,cntl3,*indices;
PetscErrorCode ierr;
/* Each DMDA manages the local vector for the portion of region 1, 2, and 3 for that processor
Each DMDA can belong on any subset (overlapping between DMDA's or not) of processors
For processes that a particular DMDA does not exist on, the corresponding comm should be set to zero
*/
DM da1 = 0,da2 = 0,da3 = 0;
/*
v1, v2, v3 represent the local vector for a single DMDA
*/
Vec vl1 = 0,vl2 = 0,vl3 = 0, vg1 = 0, vg2 = 0,vg3 = 0;
/*
globalvec and friends represent the global vectors that are used for the PETSc solvers
localvec represents the concatenation of the (up to) 3 local vectors; vl1, vl2, vl3
tovec and friends represent intermediate vectors that are ONLY used for setting up the
final communication patterns. Once this setup routine is complete they are destroyed.
The tovec is like the globalvec EXCEPT it has redundant locations for the ghost points
between regions 2+3 and 1.
*/
AO toao,globalao;
IS tois,globalis,is;
Vec tovec,globalvec,localvec;
VecScatter vscat;
PetscScalar *globalarray,*localarray,*toarray;
ierr = PetscNew(struct _p_FA,&fa);CHKERRQ(ierr);
/*
fa->sw is the stencil width
fa->p1 is the width of region 1, fa->p2 the width of region 2 (must be the same)
fa->r1 height of region 1
fa->r2 height of region 2
fa->r2 is also the height of region 3-4
(fa->p1 - fa->p2)/2 is the width of both region 3 and region 4
*/
fa->p1 = 24;
fa->p2 = 15;
fa->r1 = 6;
fa->r2 = 6;
fa->sw = 1;
fa->r1g = fa->r1 + fa->sw;
fa->r2g = fa->r2 + fa->sw;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
fa->comm[0] = PETSC_COMM_WORLD;
fa->comm[1] = PETSC_COMM_WORLD;
fa->comm[2] = PETSC_COMM_WORLD;
/* Test case with different communicators */
/* Normally one would use MPI_Comm routines to build MPI communicators on which you wish to partition the DMDAs*/
/*
if (!rank) {
fa->comm[0] = PETSC_COMM_SELF;
fa->comm[1] = 0;
fa->comm[2] = 0;
} else if (rank == 1) {
fa->comm[0] = 0;
fa->comm[1] = PETSC_COMM_SELF;
fa->comm[2] = 0;
} else {
fa->comm[0] = 0;
fa->comm[1] = 0;
fa->comm[2] = PETSC_COMM_SELF;
} */
if (fa->p2 > fa->p1 - 3) SETERRQ(PETSC_COMM_SELF,1,"Width of region fa->p2 must be at least 3 less then width of region 1");
if (!((fa->p2 - fa->p1) % 2)) SETERRQ(PETSC_COMM_SELF,1,"width of region 3 must NOT be divisible by 2");
if (fa->comm[1]) {
ierr = DMDACreate2d(fa->comm[1],DMDA_BOUNDARY_PERIODIC,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p2,fa->r2g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da2);CHKERRQ(ierr);
ierr = DMGetLocalVector(da2,&vl2);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da2,&vg2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = DMDACreate2d(fa->comm[2],DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p1-fa->p2,fa->r2g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da3);CHKERRQ(ierr);
ierr = DMGetLocalVector(da3,&vl3);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da3,&vg3);CHKERRQ(ierr);
}
if (fa->comm[0]) {
ierr = DMDACreate2d(fa->comm[0],DMDA_BOUNDARY_NONE,DMDA_BOUNDARY_NONE,DMDA_STENCIL_BOX,fa->p1,fa->r1g,PETSC_DECIDE,PETSC_DECIDE,1,fa->sw,NULL,NULL,&da1);CHKERRQ(ierr);
ierr = DMGetLocalVector(da1,&vl1);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da1,&vg1);CHKERRQ(ierr);
}
/* count the number of unknowns owned on each processor and determine the starting point of each processors ownership
for global vector with redundancy */
tonglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
tonglobal += nx*ny;
}
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Number of unknowns owned %d\n",rank,tonglobal);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
/* Get tonatural number for each node */
ierr = PetscMalloc((tonglobal+1)*sizeof(PetscInt),&tonatural);CHKERRQ(ierr);
tonglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
tonatural[tonglobal++] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) tonatural[tonglobal++] = x + i + fa->p1*(y + j);
else tonatural[tonglobal++] = fa->p2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
tonatural[tonglobal++] = fa->p1*fa->r2g + x + i + fa->p1*(y + j);
}
}
}
/* ierr = PetscIntView(tonglobal,tonatural,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
ierr = AOCreateBasic(PETSC_COMM_WORLD,tonglobal,tonatural,0,&toao);CHKERRQ(ierr);
ierr = PetscFree(tonatural);CHKERRQ(ierr);
/* count the number of unknowns owned on each processor and determine the starting point of each processors ownership
for global vector without redundancy */
fromnglobal = 0;
fa->offg[1] = 0;
offt[1] = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
offt[2] = nx*ny;
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fromnglobal += nx*ny;
fa->offg[2] = fromnglobal;
} else {
offt[2] = 0;
fa->offg[2] = 0;
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
offt[0] = offt[2] + nx*ny;
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fromnglobal += nx*ny;
fa->offg[0] = fromnglobal;
} else {
offt[0] = offt[2];
fa->offg[0] = fromnglobal;
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y == 0) ny--; /* includes the ghost points on the lower side */
fromnglobal += nx*ny;
}
ierr = MPI_Scan(&fromnglobal,&globalrstart,1,MPIU_INT,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
globalrstart -= fromnglobal;
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Number of unknowns owned %d\n",rank,fromnglobal);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD);CHKERRQ(ierr);
/* Get fromnatural number for each node */
ierr = PetscMalloc((fromnglobal+1)*sizeof(PetscInt),&fromnatural);CHKERRQ(ierr);
fromnglobal = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fa->xg[1] = x; fa->yg[1] = y; fa->mg[1] = nx; fa->ng[1] = ny;
ierr = DMDAGetGhostCorners(da2,&fa->xl[1],&fa->yl[1],0,&fa->ml[1],&fa->nl[1],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
fromnatural[fromnglobal++] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y+ny == fa->r2g) ny--; /* includes the ghost points on the upper side */
fa->xg[2] = x; fa->yg[2] = y; fa->mg[2] = nx; fa->ng[2] = ny;
ierr = DMDAGetGhostCorners(da3,&fa->xl[2],&fa->yl[2],0,&fa->ml[2],&fa->nl[2],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) fromnatural[fromnglobal++] = x + i + fa->p1*(y + j);
else fromnatural[fromnglobal++] = fa->p2 + x + i + fa->p1*(y + j);
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
if (y == 0) ny--; /* includes the ghost points on the lower side */
else y--;
fa->xg[0] = x; fa->yg[0] = y; fa->mg[0] = nx; fa->ng[0] = ny;
ierr = DMDAGetGhostCorners(da1,&fa->xl[0],&fa->yl[0],0,&fa->ml[0],&fa->nl[0],0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
fromnatural[fromnglobal++] = fa->p1*fa->r2 + x + i + fa->p1*(y + j);
}
}
}
/*ierr = PetscIntView(fromnglobal,fromnatural,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);*/
ierr = AOCreateBasic(PETSC_COMM_WORLD,fromnglobal,fromnatural,0,&globalao);CHKERRQ(ierr);
ierr = PetscFree(fromnatural);CHKERRQ(ierr);
/* ---------------------------------------------------*/
/* Create the scatter that updates 1 from 2 and 3 and 3 and 2 from 1 */
/* currently handles stencil width of 1 ONLY */
ierr = PetscMalloc(tonglobal*sizeof(PetscInt),&to);CHKERRQ(ierr);
ierr = PetscMalloc(tonglobal*sizeof(PetscInt),&from);CHKERRQ(ierr);
nscat = 0;
if (fa->comm[1]) {
ierr = DMDAGetCorners(da2,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
to[nscat] = from[nscat] = (fa->p1 - fa->p2)/2 + x + i + fa->p1*(y + j);nscat++;
}
}
}
if (fa->comm[2]) {
ierr = DMDAGetCorners(da3,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
if (x + i < (fa->p1 - fa->p2)/2) {
to[nscat] = from[nscat] = x + i + fa->p1*(y + j);nscat++;
} else {
to[nscat] = from[nscat] = fa->p2 + x + i + fa->p1*(y + j);nscat++;
}
}
}
}
if (fa->comm[0]) {
ierr = DMDAGetCorners(da1,&x,&y,0,&nx,&ny,0);CHKERRQ(ierr);
for (j=0; j<ny; j++) {
for (i=0; i<nx; i++) {
to[nscat] = fa->p1*fa->r2g + x + i + fa->p1*(y + j);
from[nscat++] = fa->p1*(fa->r2 - 1) + x + i + fa->p1*(y + j);
}
}
}
ierr = AOApplicationToPetsc(toao,nscat,to);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(globalao,nscat,from);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,nscat,to,PETSC_COPY_VALUES,&tois);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,nscat,from,PETSC_COPY_VALUES,&globalis);CHKERRQ(ierr);
ierr = PetscFree(to);CHKERRQ(ierr);
ierr = PetscFree(from);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,tonglobal,PETSC_DETERMINE,&tovec);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,fromnglobal,PETSC_DETERMINE,&globalvec);CHKERRQ(ierr);
ierr = VecScatterCreate(globalvec,globalis,tovec,tois,&vscat);CHKERRQ(ierr);
ierr = ISDestroy(&tois);CHKERRQ(ierr);
ierr = ISDestroy(&globalis);CHKERRQ(ierr);
ierr = AODestroy(&globalao);CHKERRQ(ierr);
ierr = AODestroy(&toao);CHKERRQ(ierr);
/* fill up global vector without redundant values with PETSc global numbering */
ierr = VecGetArray(globalvec,&globalarray);CHKERRQ(ierr);
for (i=0; i<fromnglobal; i++) {
globalarray[i] = globalrstart + i;
}
ierr = VecRestoreArray(globalvec,&globalarray);CHKERRQ(ierr);
/* scatter PETSc global indices to redundant valueed array */
ierr = VecScatterBegin(vscat,globalvec,tovec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat,globalvec,tovec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Create local vector that is the concatenation of the local vectors */
nlocal = 0;
cntl1 = cntl2 = cntl3 = 0;
if (fa->comm[1]) {
ierr = VecGetSize(vl2,&cntl2);CHKERRQ(ierr);
nlocal += cntl2;
}
if (fa->comm[2]) {
ierr = VecGetSize(vl3,&cntl3);CHKERRQ(ierr);
nlocal += cntl3;
}
if (fa->comm[0]) {
ierr = VecGetSize(vl1,&cntl1);CHKERRQ(ierr);
nlocal += cntl1;
}
fa->offl[0] = cntl2 + cntl3;
fa->offl[1] = 0;
fa->offl[2] = cntl2;
ierr = VecCreateSeq(PETSC_COMM_SELF,nlocal,&localvec);CHKERRQ(ierr);
/* cheat so that vl1, vl2, vl3 shared array memory with localvec */
ierr = VecGetArray(localvec,&localarray);CHKERRQ(ierr);
ierr = VecGetArray(tovec,&toarray);CHKERRQ(ierr);
if (fa->comm[1]) {
ierr = VecPlaceArray(vl2,localarray+fa->offl[1]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg2,toarray+offt[1]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da2,vg2,INSERT_VALUES,vl2);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da2,vg2,INSERT_VALUES,vl2);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da2,&vg2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = VecPlaceArray(vl3,localarray+fa->offl[2]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg3,toarray+offt[2]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da3,vg3,INSERT_VALUES,vl3);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da3,vg3,INSERT_VALUES,vl3);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da3,&vg3);CHKERRQ(ierr);
}
if (fa->comm[0]) {
ierr = VecPlaceArray(vl1,localarray+fa->offl[0]);CHKERRQ(ierr);
ierr = VecPlaceArray(vg1,toarray+offt[0]);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da1,vg1,INSERT_VALUES,vl1);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da1,vg1,INSERT_VALUES,vl1);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da1,&vg1);CHKERRQ(ierr);
}
ierr = VecRestoreArray(localvec,&localarray);CHKERRQ(ierr);
ierr = VecRestoreArray(tovec,&toarray);CHKERRQ(ierr);
/* no longer need the redundant vector and VecScatter to it */
ierr = VecScatterDestroy(&vscat);CHKERRQ(ierr);
ierr = VecDestroy(&tovec);CHKERRQ(ierr);
/* Create final scatter that goes directly from globalvec to localvec */
/* this is the one to be used in the application code */
ierr = PetscMalloc(nlocal*sizeof(PetscInt),&indices);CHKERRQ(ierr);
ierr = VecGetArray(localvec,&localarray);CHKERRQ(ierr);
for (i=0; i<nlocal; i++) {
indices[i] = (PetscInt) (localarray[i]);
}
ierr = VecRestoreArray(localvec,&localarray);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_WORLD,2,nlocal,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = PetscFree(indices);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,2*nlocal,&fa->l);CHKERRQ(ierr);
ierr = VecCreateMPI(PETSC_COMM_WORLD,2*fromnglobal,PETSC_DETERMINE,&fa->g);CHKERRQ(ierr);
ierr = VecScatterCreate(fa->g,is,fa->l,NULL,&fa->vscat);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = VecDestroy(&globalvec);CHKERRQ(ierr);
ierr = VecDestroy(&localvec);CHKERRQ(ierr);
if (fa->comm[0]) {
ierr = DMRestoreLocalVector(da1,&vl1);CHKERRQ(ierr);
ierr = DMDestroy(&da1);CHKERRQ(ierr);
}
if (fa->comm[1]) {
ierr = DMRestoreLocalVector(da2,&vl2);CHKERRQ(ierr);
ierr = DMDestroy(&da2);CHKERRQ(ierr);
}
if (fa->comm[2]) {
ierr = DMRestoreLocalVector(da3,&vl3);CHKERRQ(ierr);
ierr = DMDestroy(&da3);CHKERRQ(ierr);
}
*infa = fa;
PetscFunctionReturn(0);
}
