int main(int argc,char **argv) { PetscMPIInt rank; PetscInt m = PETSC_DECIDE,n = PETSC_DECIDE,p = PETSC_DECIDE,M = 3,N = 5,P=3,s=1; PetscInt *lx = PETSC_NULL,*ly = PETSC_NULL,*lz = PETSC_NULL; PetscErrorCode ierr; PetscTruth flg = PETSC_FALSE; DA da; Vec local,global,vslice; PetscScalar value; DAPeriodicType wrap = DA_XYPERIODIC; DAStencilType stencil_type = DA_STENCIL_BOX; VecScatter scatter; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); /* Read options */ ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-P",&P,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-p",&p,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetTruth(PETSC_NULL,"-star",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) stencil_type = DA_STENCIL_STAR; /* Create distributed array and get vectors */ ierr = DACreate3d(PETSC_COMM_WORLD,wrap,stencil_type,M,N,P,m,n,p,1,s, lx,ly,lz,&da);CHKERRQ(ierr); ierr = DAView(da,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr); ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr); ierr = GenerateSliceScatter(da,&scatter,&vslice);CHKERRQ(ierr); /* Put the value rank+1 into all locations of vslice and transfer back to global vector */ value = 1.0 + rank; ierr = VecSet(vslice,value);CHKERRQ(ierr); ierr = VecScatterBegin(scatter,vslice,global,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(scatter,vslice,global,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecView(global,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr); ierr = VecDestroy(local);CHKERRQ(ierr); ierr = VecDestroy(global);CHKERRQ(ierr); ierr = DADestroy(da);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
int main(int argc,char **argv) { PetscInt M = 13,dof=1,s=1,wrap=0,i,n,j; PetscErrorCode ierr; DA da; PetscViewer viewer; Vec local,locala,global,coors; PetscScalar *x,*alocal; PetscDraw draw; char fname[16]; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); /* Create viewers */ ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",PETSC_DECIDE,PETSC_DECIDE,600,200,&viewer);CHKERRQ(ierr); ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr); ierr = PetscDrawSetDoubleBuffer(draw);CHKERRQ(ierr); /* Read options */ ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-s",&s,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-periodic",&wrap,PETSC_NULL);CHKERRQ(ierr); /* Create distributed array and get vectors */ ierr = DACreate1d(PETSC_COMM_WORLD,(DAPeriodicType)wrap,M,dof,s,PETSC_NULL,&da);CHKERRQ(ierr); ierr = DASetUniformCoordinates(da,0.0,1.0,0.0,0.0,0.0,0.0);CHKERRQ(ierr); for (i=0; i<dof; i++) { sprintf(fname,"Field %d",(int)i); ierr = DASetFieldName(da,i,fname); } ierr = DAView(da,viewer);CHKERRQ(ierr); ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr); ierr = DACreateLocalVector(da,&locala);CHKERRQ(ierr); ierr = DAGetCoordinates(da,&coors);CHKERRQ(ierr); ierr = VecGetArray(coors,&x);CHKERRQ(ierr); /* Set values into global vectors */ ierr = VecGetArray(global,&alocal);CHKERRQ(ierr); ierr = VecGetLocalSize(global,&n);CHKERRQ(ierr); n = n/dof; for (j=0; j<dof; j++) { for (i=0; i<n; i++) { alocal[j+dof*i] = PetscSinScalar(2*PETSC_PI*(j+1)*x[i]); } } ierr = VecRestoreArray(global,&alocal);CHKERRQ(ierr); ierr = VecRestoreArray(coors,&x);CHKERRQ(ierr); ierr = VecDestroy(coords);CHKERRQ(ierr); ierr = VecView(global,viewer);CHKERRQ(ierr); /* Send ghost points to local vectors */ ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,locala);CHKERRQ(ierr); ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,locala);CHKERRQ(ierr); /* Free memory */ ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr); ierr = VecDestroy(global);CHKERRQ(ierr); ierr = VecDestroy(local);CHKERRQ(ierr); ierr = VecDestroy(locala);CHKERRQ(ierr); ierr = DADestroy(da);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
PetscErrorCode vizGA2DA() { PetscErrorCode ierr; int rank; MPI_Comm_rank(PETSC_COMM_WORLD,&rank); int d1 = 40, d2 = 50; DA da; Vec vec; const PetscInt *lx, *ly, *lz; PetscInt m,n,p; DALocalInfo info; ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR, d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr); ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr); ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr); ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr); ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr); /**/ ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr); for (int i = 0; i < m; ++i) { PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]); } for (int i = 0; i < n; ++i) { PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]); } /**/ int ga = GA_Create_handle(); int ndim = 2; int dims[2] = {d2,d1}; GA_Set_data(ga,2,dims,MT_DBL); int *map; PetscMalloc( sizeof(int)*(m+n), &map); map[0] = 0; for( int i = 1; i < n; i++ ) { map[i] = ly[i-1] + map[i-1]; } map[n] = 0; for( int i = n+1; i < m+n; i++ ) { map[i] = lx[i-n-1] + map[i-1]; } /* correct ordering, but nodeid's dont line up with mpi rank for petsc's da * DA: +---+---+ GA: +---+---+ * +-2-+-3-+ +-1-+-3-+ * +---+---+ +---+---+ * +-0-+-1-+ +-0-+-2-+ * +---+---+ +---+---+ int *map; PetscMalloc( sizeof(int)*(m+n), &map); map[0] = 0; for( int i = 1; i < m; i++ ) { map[i] = lx[i] + map[i-1]; } map[m] = 0; for( int i = m+1; i < m+n; i++ ) { map[i] = ly[i-m] + map[i-1]; } */ int block[2] = {n,m}; GA_Set_irreg_distr(ga,map,block); ierr = GA_Allocate( ga ); if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr); if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr); if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1); GA_Print_distribution(ga); int lo[2], hi[2]; NGA_Distribution(ga,rank,lo,hi); if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 || lo[0] != info.ys || hi[0] != info.ys+info.ym-1 ) { PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n", rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1); } PetscBarrier(0); PetscSynchronizedFlush(PETSC_COMM_WORLD); AO ao; DAGetAO(da,&ao); if( rank == 0 ) { int *idx, len = d1*d2; PetscReal *val; PetscMalloc(sizeof(PetscReal)*len, &val); PetscMalloc(sizeof(int)*len, &idx); for (int j = 0; j < d2; ++j) { for (int i = 0; i < d1; ++i) { idx[i + d1*j] = i + d1*j; val[i + d1*j] = i + d1*j; } } AOApplicationToPetsc(ao,len,idx); VecSetValues(vec,len,idx,val,INSERT_VALUES); int a[2], b[2],ld[1]={0}; double c = 0; for (int j = 0; j < d2; ++j) { for (int i = 0; i < d1; ++i) { a[0] = j; a[1] = i; // printf("%5.0f ",c); NGA_Put(ga,a,a,&c,ld); c++; } } } // GA_Print(ga); VecAssemblyBegin(vec); VecAssemblyEnd(vec); int ld; double *ptr; NGA_Access(ga,lo,hi,&ptr,&ld); PetscReal **d; int c=0; ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr); for (int j = info.ys; j < info.ys+info.ym; ++j) { for (int i = info.xs; i < info.xs+info.xm; ++i) { if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] ) GA_Error("DA array is not equal to GA array",1); // printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]); c++; } } ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr); c=0; PetscReal *v; int start, end; VecGetOwnershipRange(vec, &start, &end); VecGetArray( vec, &v ); for( int i = start; i < end; i++) { // printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") ); } VecRestoreArray( vec, &v ); NGA_Release_update(ga,lo,hi); Vec gada; VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada); VecView(gada,PETSC_VIEWER_STDOUT_SELF); GA_Destroy(ga); ierr = VecDestroy(vec); CHKERRQ(ierr); ierr = DADestroy(da); CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **argv) { PetscMPIInt rank; PetscInt M = -10,N = -8; PetscErrorCode ierr; PetscTruth flg = PETSC_FALSE; DA da; PetscViewer viewer; Vec local,global; PetscScalar value; DAPeriodicType ptype = DA_NONPERIODIC; DAStencilType stype = DA_STENCIL_BOX; #if defined(PETSC_HAVE_MATLAB_ENGINE) PetscViewer mviewer; #endif ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,300,300,&viewer);CHKERRQ(ierr); #if defined(PETSC_HAVE_MATLAB_ENGINE) ierr = PetscViewerMatlabOpen(PETSC_COMM_WORLD,"tmp.mat",FILE_MODE_WRITE,&mviewer);CHKERRQ(ierr); #endif ierr = PetscOptionsGetTruth(PETSC_NULL,"-star_stencil",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) stype = DA_STENCIL_STAR; /* Create distributed array and get vectors */ ierr = DACreate2d(PETSC_COMM_WORLD,ptype,stype,M,N,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr); ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr); value = -3.0; ierr = VecSet(global,value);CHKERRQ(ierr); ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); value = rank+1; ierr = VecScale(local,value);CHKERRQ(ierr); ierr = DALocalToGlobal(da,local,ADD_VALUES,global);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetTruth(PETSC_NULL, "-view_global", &flg,PETSC_NULL);CHKERRQ(ierr); if (flg) { /* view global vector in natural ordering */ ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } ierr = DAView(da,viewer);CHKERRQ(ierr); ierr = VecView(global,viewer);CHKERRQ(ierr); #if defined(PETSC_HAVE_MATLAB_ENGINE) ierr = DAView(da,mviewer);CHKERRQ(ierr); ierr = VecView(global,mviewer);CHKERRQ(ierr); #endif /* Free memory */ #if defined(PETSC_HAVE_MATLAB_ENGINE) ierr = PetscViewerDestroy(mviewer);CHKERRQ(ierr); #endif ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr); ierr = VecDestroy(local);CHKERRQ(ierr); ierr = VecDestroy(global);CHKERRQ(ierr); ierr = DADestroy(da);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }