int main(int argc,char **argv) { DM da; /* distributed array */ Vec x,b,u; /* approx solution, RHS, exact solution */ Mat A; /* linear system matrix */ KSP ksp; /* linear solver context */ PetscRandom rctx; /* random number generator context */ PetscReal norm; /* norm of solution error */ PetscInt i,j,its; PetscErrorCode ierr; PetscBool flg = PETSC_FALSE; PetscLogStage stage; DMDALocalInfo info; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); /* Create distributed array to handle parallel distribution. The problem size will default to 8 by 7, but this can be changed using -da_grid_x M -da_grid_y N */ ierr = DMDACreate2d(PETSC_COMM_WORLD, DMDA_BOUNDARY_NONE, DMDA_BOUNDARY_NONE,DMDA_STENCIL_STAR,-8,-7,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Compute the matrix and right-hand-side vector that define the linear system, Ax = b. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create parallel matrix preallocated according to the DMDA, format AIJ by default. To use symmetric storage, run with -dm_mat_type sbaij -mat_ignore_lower_triangular */ ierr = DMCreateMatrix(da,MATAIJ,&A);CHKERRQ(ierr); /* Set matrix elements for the 2-D, five-point stencil in parallel. - Each processor needs to insert only elements that it owns locally (but any non-local elements will be sent to the appropriate processor during matrix assembly). - Rows and columns are specified by the stencil - Entries are normalized for a domain [0,1]x[0,1] */ ierr = PetscLogStageRegister("Assembly", &stage);CHKERRQ(ierr); ierr = PetscLogStagePush(stage);CHKERRQ(ierr); ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr); for (j=info.ys; j<info.ys+info.ym; j++) { for (i=info.xs; i<info.xs+info.xm; i++) { PetscReal hx = 1./info.mx,hy = 1./info.my; MatStencil row = {0},col[5] = {{0}}; PetscScalar v[5]; PetscInt ncols = 0; row.j = j; row.i = i; col[ncols].j = j; col[ncols].i = i; v[ncols++] = 2*(hx/hy + hy/hx); /* boundaries */ if (i>0) {col[ncols].j = j; col[ncols].i = i-1; v[ncols++] = -hy/hx;} if (i<info.mx-1) {col[ncols].j = j; col[ncols].i = i+1; v[ncols++] = -hy/hx;} if (j>0) {col[ncols].j = j-1; col[ncols].i = i; v[ncols++] = -hx/hy;} if (j<info.my-1) {col[ncols].j = j+1; col[ncols].i = i; v[ncols++] = -hx/hy;} ierr = MatSetValuesStencil(A,1,&row,ncols,col,v,INSERT_VALUES);CHKERRQ(ierr); } } /* Assemble matrix, using the 2-step process: MatAssemblyBegin(), MatAssemblyEnd() Computations can be done while messages are in transition by placing code between these two statements. */ ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = PetscLogStagePop();CHKERRQ(ierr); /* Create parallel vectors compatible with the DMDA. */ ierr = DMCreateGlobalVector(da,&u);CHKERRQ(ierr); ierr = VecDuplicate(u,&b);CHKERRQ(ierr); ierr = VecDuplicate(u,&x);CHKERRQ(ierr); /* Set exact solution; then compute right-hand-side vector. By default we use an exact solution of a vector with all elements of 1.0; Alternatively, using the runtime option -random_sol forms a solution vector with random components. */ ierr = PetscOptionsGetBool(PETSC_NULL,"-random_exact_sol",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) { ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr); ierr = VecSetRandom(u,rctx);CHKERRQ(ierr); ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr); } else { ierr = VecSet(u,1.);CHKERRQ(ierr); } ierr = MatMult(A,u,b);CHKERRQ(ierr); /* View the exact solution vector if desired */ flg = PETSC_FALSE; ierr = PetscOptionsGetBool(PETSC_NULL,"-view_exact_sol",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) {ierr = VecView(u,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);} /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create the linear solver and set various options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create linear solver context */ ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr); /* Set operators. Here the matrix that defines the linear system also serves as the preconditioning matrix. */ ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr); /* Set runtime options, e.g., -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol> These options will override those specified above as long as KSPSetFromOptions() is called _after_ any other customization routines. */ ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Solve the linear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Check solution and clean up - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Check the error */ ierr = VecAXPY(x,-1.,u);CHKERRQ(ierr); ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr); ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr); /* Print convergence information. PetscPrintf() produces a single print statement from all processes that share a communicator. An alternative is PetscFPrintf(), which prints to a file. */ ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %G iterations %D\n",norm,its);CHKERRQ(ierr); /* Free work space. All PETSc objects should be destroyed when they are no longer needed. */ ierr = KSPDestroy(&ksp);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); /* Always call PetscFinalize() before exiting a program. This routine - finalizes the PETSc libraries as well as MPI - provides summary and diagnostic information if certain runtime options are chosen (e.g., -log_summary). */ ierr = PetscFinalize(); return 0; }
static PetscErrorCode SNESComputeJacobian_DMDA(SNES snes,Vec X,Mat *A,Mat *B,MatStructure *mstr,void *ctx) { PetscErrorCode ierr; DM dm; DMSNES_DA *dmdasnes = (DMSNES_DA*)ctx; DMDALocalInfo info; Vec Xloc; void *x; PetscFunctionBegin; if (!dmdasnes->residuallocal) SETERRQ(PetscObjectComm((PetscObject)snes),PETSC_ERR_PLIB,"Corrupt context"); ierr = SNESGetDM(snes,&dm);CHKERRQ(ierr); if (dmdasnes->jacobianlocal) { ierr = DMGetLocalVector(dm,&Xloc);CHKERRQ(ierr); ierr = DMGlobalToLocalBegin(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(dm,X,INSERT_VALUES,Xloc);CHKERRQ(ierr); ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr); ierr = DMDAVecGetArray(dm,Xloc,&x);CHKERRQ(ierr); CHKMEMQ; ierr = (*dmdasnes->jacobianlocal)(&info,x,*A,*B,mstr,dmdasnes->jacobianlocalctx);CHKERRQ(ierr); CHKMEMQ; ierr = DMDAVecRestoreArray(dm,Xloc,&x);CHKERRQ(ierr); ierr = DMRestoreLocalVector(dm,&Xloc);CHKERRQ(ierr); } else { MatFDColoring fdcoloring; ierr = PetscObjectQuery((PetscObject)dm,"DMDASNES_FDCOLORING",(PetscObject*)&fdcoloring);CHKERRQ(ierr); if (!fdcoloring) { ISColoring coloring; ierr = DMCreateColoring(dm,dm->coloringtype,&coloring);CHKERRQ(ierr); ierr = MatFDColoringCreate(*B,coloring,&fdcoloring);CHKERRQ(ierr); ierr = ISColoringDestroy(&coloring);CHKERRQ(ierr); switch (dm->coloringtype) { case IS_COLORING_GLOBAL: ierr = MatFDColoringSetFunction(fdcoloring,(PetscErrorCode (*)(void))SNESComputeFunction_DMDA,dmdasnes);CHKERRQ(ierr); break; default: SETERRQ1(PetscObjectComm((PetscObject)snes),PETSC_ERR_SUP,"No support for coloring type '%s'",ISColoringTypes[dm->coloringtype]); } ierr = PetscObjectSetOptionsPrefix((PetscObject)fdcoloring,((PetscObject)dm)->prefix);CHKERRQ(ierr); ierr = MatFDColoringSetFromOptions(fdcoloring);CHKERRQ(ierr); ierr = PetscObjectCompose((PetscObject)dm,"DMDASNES_FDCOLORING",(PetscObject)fdcoloring);CHKERRQ(ierr); ierr = PetscObjectDereference((PetscObject)fdcoloring);CHKERRQ(ierr); /* The following breaks an ugly reference counting loop that deserves a paragraph. MatFDColoringApply() will call * VecDuplicate() with the state Vec and store inside the MatFDColoring. This Vec will duplicate the Vec, but the * MatFDColoring is composed with the DM. We dereference the DM here so that the reference count will eventually * drop to 0. Note the code in DMDestroy() that exits early for a negative reference count. That code path will be * taken when the PetscObjectList for the Vec inside MatFDColoring is destroyed. */ ierr = PetscObjectDereference((PetscObject)dm);CHKERRQ(ierr); } *mstr = SAME_NONZERO_PATTERN; ierr = MatFDColoringApply(*B,fdcoloring,X,mstr,snes);CHKERRQ(ierr); } /* This will be redundant if the user called both, but it's too common to forget. */ if (*A != *B) { ierr = MatAssemblyBegin(*A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(*A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); } PetscFunctionReturn(0); }
PetscErrorCode DMView_DA_2d(DM da,PetscViewer viewer) { PetscErrorCode ierr; PetscMPIInt rank; PetscBool iascii,isdraw,isbinary; DM_DA *dd = (DM_DA*)da->data; #if defined(PETSC_HAVE_MATLAB_ENGINE) PetscBool ismatlab; #endif PetscFunctionBegin; ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)da),&rank);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERBINARY,&isbinary);CHKERRQ(ierr); #if defined(PETSC_HAVE_MATLAB_ENGINE) ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERMATLAB,&ismatlab);CHKERRQ(ierr); #endif if (iascii) { PetscViewerFormat format; ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr); if (format != PETSC_VIEWER_ASCII_VTK && format != PETSC_VIEWER_ASCII_VTK_CELL) { DMDALocalInfo info; ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr); ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);CHKERRQ(ierr); ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %D N %D m %D n %D w %D s %D\n",rank,dd->M,dd->N,dd->m,dd->n,dd->w,dd->s);CHKERRQ(ierr); ierr = PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %D %D, Y range of indices: %D %D\n",info.xs,info.xs+info.xm,info.ys,info.ys+info.ym);CHKERRQ(ierr); ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);CHKERRQ(ierr); } else { ierr = DMView_DA_VTK(da,viewer);CHKERRQ(ierr); } } else if (isdraw) { PetscDraw draw; double ymin = -1*dd->s-1,ymax = dd->N+dd->s; double xmin = -1*dd->s-1,xmax = dd->M+dd->s; double x,y; PetscInt base; const PetscInt *idx; char node[10]; PetscBool isnull; ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr); ierr = PetscDrawIsNull(draw,&isnull);CHKERRQ(ierr); if (isnull) PetscFunctionReturn(0); if (!da->coordinates) { ierr = PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);CHKERRQ(ierr); } ierr = PetscDrawSynchronizedClear(draw);CHKERRQ(ierr); /* first processor draw all node lines */ if (!rank) { ymin = 0.0; ymax = dd->N - 1; for (xmin=0; xmin<dd->M; xmin++) { ierr = PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);CHKERRQ(ierr); } xmin = 0.0; xmax = dd->M - 1; for (ymin=0; ymin<dd->N; ymin++) { ierr = PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);CHKERRQ(ierr); } } ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr); ierr = PetscDrawPause(draw);CHKERRQ(ierr); /* draw my box */ ymin = dd->ys; ymax = dd->ye - 1; xmin = dd->xs/dd->w; xmax =(dd->xe-1)/dd->w; ierr = PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);CHKERRQ(ierr); ierr = PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);CHKERRQ(ierr); ierr = PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);CHKERRQ(ierr); ierr = PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);CHKERRQ(ierr); /* put in numbers */ base = (dd->base)/dd->w; for (y=ymin; y<=ymax; y++) { for (x=xmin; x<=xmax; x++) { sprintf(node,"%d",(int)base++); ierr = PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);CHKERRQ(ierr); } } ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr); ierr = PetscDrawPause(draw);CHKERRQ(ierr); /* overlay ghost numbers, useful for error checking */ /* put in numbers */ base = 0; ierr = ISLocalToGlobalMappingGetIndices(da->ltogmapb,&idx);CHKERRQ(ierr); ymin = dd->Ys; ymax = dd->Ye; xmin = dd->Xs; xmax = dd->Xe; for (y=ymin; y<ymax; y++) { for (x=xmin; x<xmax; x++) { if ((base % dd->w) == 0) { sprintf(node,"%d",(int)(idx[base])); ierr = PetscDrawString(draw,x/dd->w,y,PETSC_DRAW_BLUE,node);CHKERRQ(ierr); } base++; } } ierr = ISLocalToGlobalMappingRestoreIndices(da->ltogmapb,&idx); ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr); ierr = PetscDrawPause(draw);CHKERRQ(ierr); } else if (isbinary) { ierr = DMView_DA_Binary(da,viewer);CHKERRQ(ierr); #if defined(PETSC_HAVE_MATLAB_ENGINE) } else if (ismatlab) { ierr = DMView_DA_Matlab(da,viewer);CHKERRQ(ierr); #endif } PetscFunctionReturn(0); }
PetscErrorCode FormMatrix(DM da,Mat jac) { PetscErrorCode ierr; PetscInt i,j,nrows = 0; MatStencil col[5],row,*rows; PetscScalar v[5],hx,hy,hxdhy,hydhx; DMDALocalInfo info; PetscFunctionBeginUser; ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr); hx = 1.0/(PetscReal)(info.mx-1); hy = 1.0/(PetscReal)(info.my-1); hxdhy = hx/hy; hydhx = hy/hx; ierr = PetscMalloc(info.ym*info.xm*sizeof(MatStencil),&rows);CHKERRQ(ierr); /* Compute entries for the locally owned part of the Jacobian. - Currently, all PETSc parallel matrix formats are partitioned by contiguous chunks of rows across the processors. - Each processor needs to insert only elements that it owns locally (but any non-local elements will be sent to the appropriate processor during matrix assembly). - Here, we set all entries for a particular row at once. - We can set matrix entries either using either MatSetValuesLocal() or MatSetValues(), as discussed above. */ for (j=info.ys; j<info.ys+info.ym; j++) { for (i=info.xs; i<info.xs+info.xm; i++) { row.j = j; row.i = i; /* boundary points */ if (i == 0 || j == 0 || i == info.mx-1 || j == info.my-1) { v[0] = 2.0*(hydhx + hxdhy); ierr = MatSetValuesStencil(jac,1,&row,1,&row,v,INSERT_VALUES);CHKERRQ(ierr); rows[nrows].i = i; rows[nrows++].j = j; } else { /* interior grid points */ v[0] = -hxdhy; col[0].j = j - 1; col[0].i = i; v[1] = -hydhx; col[1].j = j; col[1].i = i-1; v[2] = 2.0*(hydhx + hxdhy); col[2].j = row.j; col[2].i = row.i; v[3] = -hydhx; col[3].j = j; col[3].i = i+1; v[4] = -hxdhy; col[4].j = j + 1; col[4].i = i; ierr = MatSetValuesStencil(jac,1,&row,5,col,v,INSERT_VALUES);CHKERRQ(ierr); } } } /* Assemble matrix, using the 2-step process: MatAssemblyBegin(), MatAssemblyEnd(). */ ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatZeroRowsColumnsStencil(jac,nrows,rows,2.0*(hydhx + hxdhy),NULL,NULL);CHKERRQ(ierr); ierr = PetscFree(rows);CHKERRQ(ierr); /* Tell the matrix we will never add a new nonzero location to the matrix. If we do, it will generate an error. */ ierr = MatSetOption(jac,MAT_NEW_NONZERO_LOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr); PetscFunctionReturn(0); }
static PetscErrorCode DMDAVTKWriteAll_VTS(DM da,PetscViewer viewer) { #if defined(PETSC_USE_REAL_SINGLE) const char precision[] = "Float32"; #elif defined(PETSC_USE_REAL_DOUBLE) const char precision[] = "Float64"; #else const char precision[] = "UnknownPrecision"; #endif MPI_Comm comm; PetscViewer_VTK *vtk = (PetscViewer_VTK*)viewer->data; PetscViewerVTKObjectLink link; FILE *fp; PetscMPIInt rank,size,tag; DMDALocalInfo info; PetscInt dim,mx,my,mz,bs,boffset,maxnnodes,i,j,k,f,r; PetscInt rloc[6],(*grloc)[6] = NULL; PetscScalar *array,*array2; PetscReal gmin[3],gmax[3]; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr); #if defined(PETSC_USE_COMPLEX) SETERRQ(comm,PETSC_ERR_SUP,"Complex values not supported"); #endif ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); ierr = DMDAGetInfo(da,&dim, &mx,&my,&mz, 0,0,0, &bs,0,0,0,0,0);CHKERRQ(ierr); ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr); ierr = DMDAGetBoundingBox(da,gmin,gmax);CHKERRQ(ierr); ierr = PetscFOpen(comm,vtk->filename,"wb",&fp);CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp,"<?xml version=\"1.0\"?>\n");CHKERRQ(ierr); #if defined(PETSC_WORDS_BIGENDIAN) ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"StructuredGrid\" version=\"0.1\" byte_order=\"BigEndian\">\n");CHKERRQ(ierr); #else ierr = PetscFPrintf(comm,fp,"<VTKFile type=\"StructuredGrid\" version=\"0.1\" byte_order=\"LittleEndian\">\n");CHKERRQ(ierr); #endif ierr = PetscFPrintf(comm,fp," <StructuredGrid WholeExtent=\"%D %D %D %D %D %D\">\n",0,mx-1,0,my-1,0,mz-1);CHKERRQ(ierr); if (!rank) {ierr = PetscMalloc(size*6*sizeof(PetscInt),&grloc);CHKERRQ(ierr);} rloc[0] = info.xs; rloc[1] = info.xm; rloc[2] = info.ys; rloc[3] = info.ym; rloc[4] = info.zs; rloc[5] = info.zm; ierr = MPI_Gather(rloc,6,MPIU_INT,&grloc[0][0],6,MPIU_INT,0,comm);CHKERRQ(ierr); /* Write XML header */ maxnnodes = 0; /* Used for the temporary array size on rank 0 */ boffset = 0; /* Offset into binary file */ for (r=0; r<size; r++) { PetscInt xs=-1,xm=-1,ys=-1,ym=-1,zs=-1,zm=-1,nnodes = 0; if (!rank) { xs = grloc[r][0]; xm = grloc[r][1]; ys = grloc[r][2]; ym = grloc[r][3]; zs = grloc[r][4]; zm = grloc[r][5]; nnodes = xm*ym*zm; } maxnnodes = PetscMax(maxnnodes,nnodes); #if 0 switch (dim) { case 1: ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,0,0,0,0);CHKERRQ(ierr); break; case 2: ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm,ys+ym-1,xs,xs+xm-1,0,0);CHKERRQ(ierr); break; case 3: ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,ys,ys+ym-1,zs,zs+zm-1);CHKERRQ(ierr); break; default: SETERRQ1(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"No support for dimension %D",dim); } #endif ierr = PetscFPrintf(comm,fp," <Piece Extent=\"%D %D %D %D %D %D\">\n",xs,xs+xm-1,ys,ys+ym-1,zs,zs+zm-1);CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp," <Points>\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"Position\" NumberOfComponents=\"3\" format=\"appended\" offset=\"%D\" />\n",precision,boffset);CHKERRQ(ierr); boffset += 3*nnodes*sizeof(PetscScalar) + sizeof(int); ierr = PetscFPrintf(comm,fp," </Points>\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp," <PointData Scalars=\"ScalarPointData\">\n");CHKERRQ(ierr); for (link=vtk->link; link; link=link->next) { Vec X = (Vec)link->vec; const char *vecname = ""; if (((PetscObject)X)->name || link != vtk->link) { /* If the object is already named, use it. If it is past the first link, name it to disambiguate. */ ierr = PetscObjectGetName((PetscObject)X,&vecname);CHKERRQ(ierr); } for (i=0; i<bs; i++) { char buf[256]; const char *fieldname; ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr); if (!fieldname) { ierr = PetscSNPrintf(buf,sizeof(buf),"Unnamed%D",i);CHKERRQ(ierr); fieldname = buf; } ierr = PetscFPrintf(comm,fp," <DataArray type=\"%s\" Name=\"%s%s\" NumberOfComponents=\"1\" format=\"appended\" offset=\"%D\" />\n",precision,vecname,fieldname,boffset);CHKERRQ(ierr); boffset += nnodes*sizeof(PetscScalar) + sizeof(int); } } ierr = PetscFPrintf(comm,fp," </PointData>\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp," </Piece>\n");CHKERRQ(ierr); } ierr = PetscFPrintf(comm,fp," </StructuredGrid>\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp," <AppendedData encoding=\"raw\">\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp,"_");CHKERRQ(ierr); /* Now write the arrays. */ tag = ((PetscObject)viewer)->tag; ierr = PetscMalloc2(maxnnodes*PetscMax(3,bs),PetscScalar,&array,maxnnodes*3,PetscScalar,&array2);CHKERRQ(ierr); for (r=0; r<size; r++) { MPI_Status status; PetscInt xs=-1,xm=-1,ys=-1,ym=-1,zs=-1,zm=-1,nnodes = 0; if (!rank) { xs = grloc[r][0]; xm = grloc[r][1]; ys = grloc[r][2]; ym = grloc[r][3]; zs = grloc[r][4]; zm = grloc[r][5]; nnodes = xm*ym*zm; } else if (r == rank) { nnodes = info.xm*info.ym*info.zm; } { /* Write the coordinates */ Vec Coords; ierr = DMGetCoordinates(da,&Coords);CHKERRQ(ierr); if (Coords) { const PetscScalar *coords; ierr = VecGetArrayRead(Coords,&coords);CHKERRQ(ierr); if (!rank) { if (r) { PetscMPIInt nn; ierr = MPI_Recv(array,nnodes*dim,MPIU_SCALAR,r,tag,comm,&status);CHKERRQ(ierr); ierr = MPI_Get_count(&status,MPIU_SCALAR,&nn);CHKERRQ(ierr); if (nn != nnodes*dim) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Array size mismatch"); } else { ierr = PetscMemcpy(array,coords,nnodes*dim*sizeof(PetscScalar));CHKERRQ(ierr); } /* Transpose coordinates to VTK (C-style) ordering */ for (k=0; k<zm; k++) { for (j=0; j<ym; j++) { for (i=0; i<xm; i++) { PetscInt Iloc = i+xm*(j+ym*k); array2[Iloc*3+0] = array[Iloc*dim + 0]; array2[Iloc*3+1] = dim > 1 ? array[Iloc*dim + 1] : 0; array2[Iloc*3+2] = dim > 2 ? array[Iloc*dim + 2] : 0; } } } } else if (r == rank) { ierr = MPI_Send((void*)coords,nnodes*dim,MPIU_SCALAR,0,tag,comm);CHKERRQ(ierr); } ierr = VecRestoreArrayRead(Coords,&coords);CHKERRQ(ierr); } else { /* Fabricate some coordinates using grid index */ for (k=0; k<zm; k++) { for (j=0; j<ym; j++) { for (i=0; i<xm; i++) { PetscInt Iloc = i+xm*(j+ym*k); array2[Iloc*3+0] = xs+i; array2[Iloc*3+1] = ys+j; array2[Iloc*3+2] = zs+k; } } } } ierr = PetscViewerVTKFWrite(viewer,fp,array2,nnodes*3,PETSC_SCALAR);CHKERRQ(ierr); } /* Write each of the objects queued up for this file */ for (link=vtk->link; link; link=link->next) { Vec X = (Vec)link->vec; const PetscScalar *x; ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr); if (!rank) { if (r) { PetscMPIInt nn; ierr = MPI_Recv(array,nnodes*bs,MPIU_SCALAR,r,tag,comm,&status);CHKERRQ(ierr); ierr = MPI_Get_count(&status,MPIU_SCALAR,&nn);CHKERRQ(ierr); if (nn != nnodes*bs) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Array size mismatch receiving from rank %D",r); } else { ierr = PetscMemcpy(array,x,nnodes*bs*sizeof(PetscScalar));CHKERRQ(ierr); } for (f=0; f<bs; f++) { /* Extract and transpose the f'th field */ for (k=0; k<zm; k++) { for (j=0; j<ym; j++) { for (i=0; i<xm; i++) { PetscInt Iloc = i+xm*(j+ym*k); array2[Iloc] = array[Iloc*bs + f]; } } } ierr = PetscViewerVTKFWrite(viewer,fp,array2,nnodes,PETSC_SCALAR);CHKERRQ(ierr); } } else if (r == rank) { ierr = MPI_Send((void*)x,nnodes*bs,MPIU_SCALAR,0,tag,comm);CHKERRQ(ierr); } ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr); } } ierr = PetscFree2(array,array2);CHKERRQ(ierr); ierr = PetscFree(grloc);CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp,"\n </AppendedData>\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm,fp,"</VTKFile>\n");CHKERRQ(ierr); ierr = PetscFClose(comm,fp);CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **argv) { PetscErrorCode ierr; DM da,*subda; PetscInt i,dim=3; PetscMPIInt size,rank; Vec v; Vec slvec,sgvec; IS *ois,*iis; VecScatter oscata; VecScatter *iscat,*oscat,*gscat; DMDALocalInfo info; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,"-dim",&dim,NULL);CHKERRQ(ierr); /* Create distributed array and get vectors */ ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); if (dim == 2) { ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,&da);CHKERRQ(ierr); } else if (dim == 3) { ierr = DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,-4,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr); } ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr); ierr = DMCreateDomainDecomposition(da,NULL,NULL,&iis,&ois,&subda);CHKERRQ(ierr); ierr = DMCreateDomainDecompositionScatters(da,1,subda,&iscat,&oscat,&gscat);CHKERRQ(ierr); { DMDALocalInfo subinfo; MatStencil lower,upper; IS patchis,subpatchis; Vec smallvec; Vec largevec; VecScatter patchscat; ierr = DMDAGetLocalInfo(subda[0],&subinfo);CHKERRQ(ierr); lower.i = info.xs; lower.j = info.ys; lower.k = info.zs; upper.i = info.xs+info.xm; upper.j = info.ys+info.ym; upper.k = info.zs+info.zm; /* test the patch IS as a thing to scatter to/from */ ierr = DMDACreatePatchIS(da,&lower,&upper,&patchis);CHKERRQ(ierr); ierr = DMGetGlobalVector(da,&largevec);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_SELF,&smallvec);CHKERRQ(ierr); ierr = VecSetSizes(smallvec,info.dof*(upper.i - lower.i)*(upper.j - lower.j)*(upper.k - lower.k),PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(smallvec);CHKERRQ(ierr); ierr = VecScatterCreate(smallvec,NULL,largevec,patchis,&patchscat);CHKERRQ(ierr); ierr = FillLocalSubdomain(subda[0],smallvec);CHKERRQ(ierr); ierr = VecSet(largevec,0);CHKERRQ(ierr); ierr = VecScatterBegin(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); for (i = 0; i < size; i++) { if (i == rank) { ierr = ISView(patchis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); ierr = VecScatterView(patchscat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); ierr = VecView(smallvec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); } ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); } ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); ierr = VecView(largevec,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = VecDestroy(&smallvec);CHKERRQ(ierr); ierr = DMRestoreGlobalVector(da,&largevec);CHKERRQ(ierr); ierr = ISDestroy(&patchis);CHKERRQ(ierr); ierr = VecScatterDestroy(&patchscat);CHKERRQ(ierr); } /* view the various parts */ { for (i = 0; i < size; i++) { if (i == rank) { ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr); ierr = DMView(subda[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); } ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); } ierr = DMGetLocalVector(subda[0],&slvec);CHKERRQ(ierr); ierr = DMGetGlobalVector(subda[0],&sgvec);CHKERRQ(ierr); ierr = DMGetGlobalVector(da,&v);CHKERRQ(ierr); /* test filling outer between the big DM and the small ones with the IS scatter*/ ierr = VecScatterCreate(v,ois[0],sgvec,NULL,&oscata);CHKERRQ(ierr); ierr = FillLocalSubdomain(subda[0],sgvec);CHKERRQ(ierr); ierr = VecScatterBegin(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); /* test the local-to-local scatter */ /* fill up the local subdomain and then add them together */ ierr = FillLocalSubdomain(da,v);CHKERRQ(ierr); ierr = VecScatterBegin(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* test ghost scattering backwards */ ierr = VecSet(v,0);CHKERRQ(ierr); ierr = VecScatterBegin(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* test overlap scattering backwards */ ierr = DMLocalToGlobalBegin(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr); ierr = DMLocalToGlobalEnd(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr); ierr = VecSet(v,0);CHKERRQ(ierr); ierr = VecScatterBegin(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* test interior scattering backwards */ ierr = VecSet(v,0);CHKERRQ(ierr); ierr = VecScatterBegin(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* test matrix allocation */ for (i = 0; i < size; i++) { if (i == rank) { Mat m; ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr); ierr = DMSetMatType(subda[0],MATAIJ);CHKERRQ(ierr); ierr = DMCreateMatrix(subda[0],&m);CHKERRQ(ierr); ierr = MatView(m,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); ierr = MatDestroy(&m);CHKERRQ(ierr); } ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); } ierr = DMRestoreLocalVector(subda[0],&slvec);CHKERRQ(ierr); ierr = DMRestoreGlobalVector(subda[0],&sgvec);CHKERRQ(ierr); ierr = DMRestoreGlobalVector(da,&v);CHKERRQ(ierr); } ierr = DMDestroy(&subda[0]);CHKERRQ(ierr); ierr = ISDestroy(&ois[0]);CHKERRQ(ierr); ierr = ISDestroy(&iis[0]);CHKERRQ(ierr); ierr = VecScatterDestroy(&iscat[0]);CHKERRQ(ierr); ierr = VecScatterDestroy(&oscat[0]);CHKERRQ(ierr); ierr = VecScatterDestroy(&gscat[0]);CHKERRQ(ierr); ierr = VecScatterDestroy(&oscata);CHKERRQ(ierr); ierr = PetscFree(iscat);CHKERRQ(ierr); ierr = PetscFree(oscat);CHKERRQ(ierr); ierr = PetscFree(gscat);CHKERRQ(ierr); ierr = PetscFree(oscata);CHKERRQ(ierr); ierr = PetscFree(subda);CHKERRQ(ierr); ierr = PetscFree(ois);CHKERRQ(ierr); ierr = PetscFree(iis);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
PetscErrorCode NonlinearGS(SNES snes, Vec X, Vec B, void *ctx) { DMDALocalInfo info; Field **x,**b; PetscErrorCode ierr; Vec localX, localB; DM da; PetscInt xints,xinte,yints,yinte,i,j,k,l; PetscInt max_its,tot_its; PetscInt sweeps; PetscReal rtol,atol,stol; PetscReal hx,hy,dhx,dhy,hxdhy,hydhx; PetscReal grashof,prandtl,lid; PetscScalar u,uxx,uyy,vx,vy,avx,avy,vxp,vxm,vyp,vym; PetscScalar fu, fv, fomega, ftemp; PetscScalar dfudu; PetscScalar dfvdv; PetscScalar dfodu, dfodv, dfodo; PetscScalar dftdu, dftdv, dftdt; PetscScalar yu=0, yv=0, yo=0, yt=0; PetscScalar bjiu, bjiv, bjiomega, bjitemp; PetscBool ptconverged; PetscReal pfnorm,pfnorm0,pynorm,pxnorm; AppCtx *user = (AppCtx*)ctx; PetscFunctionBeginUser; grashof = user->grashof; prandtl = user->prandtl; lid = user->lidvelocity; tot_its = 0; ierr = SNESNGSGetTolerances(snes,&rtol,&atol,&stol,&max_its);CHKERRQ(ierr); ierr = SNESNGSGetSweeps(snes,&sweeps);CHKERRQ(ierr); ierr = SNESGetDM(snes,(DM*)&da);CHKERRQ(ierr); ierr = DMGetLocalVector(da,&localX);CHKERRQ(ierr); if (B) { ierr = DMGetLocalVector(da,&localB);CHKERRQ(ierr); } /* Scatter ghost points to local vector, using the 2-step process DMGlobalToLocalBegin(), DMGlobalToLocalEnd(). */ ierr = DMGlobalToLocalBegin(da,X,INSERT_VALUES,localX);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(da,X,INSERT_VALUES,localX);CHKERRQ(ierr); if (B) { ierr = DMGlobalToLocalBegin(da,B,INSERT_VALUES,localB);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(da,B,INSERT_VALUES,localB);CHKERRQ(ierr); } ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr); ierr = DMDAVecGetArray(da,localX,&x);CHKERRQ(ierr); if (B) { ierr = DMDAVecGetArrayRead(da,localB,&b);CHKERRQ(ierr); } /* looks like a combination of the formfunction / formjacobian routines */ dhx = (PetscReal)(info.mx-1);dhy = (PetscReal)(info.my-1); hx = 1.0/dhx; hy = 1.0/dhy; hxdhy = hx*dhy; hydhx = hy*dhx; xints = info.xs; xinte = info.xs+info.xm; yints = info.ys; yinte = info.ys+info.ym; /* Set the boundary conditions on the momentum equations */ /* Test whether we are on the bottom edge of the global array */ if (yints == 0) { j = 0; /* bottom edge */ for (i=info.xs; i<info.xs+info.xm; i++) { if (B) { bjiu = b[j][i].u; bjiv = b[j][i].v; } else { bjiu = 0.0; bjiv = 0.0; } x[j][i].u = 0.0 + bjiu; x[j][i].v = 0.0 + bjiv; } } /* Test whether we are on the top edge of the global array */ if (yinte == info.my) { j = info.my - 1; /* top edge */ for (i=info.xs; i<info.xs+info.xm; i++) { if (B) { bjiu = b[j][i].u; bjiv = b[j][i].v; } else { bjiu = 0.0; bjiv = 0.0; } x[j][i].u = lid + bjiu; x[j][i].v = bjiv; } } /* Test whether we are on the left edge of the global array */ if (xints == 0) { i = 0; /* left edge */ for (j=info.ys; j<info.ys+info.ym; j++) { if (B) { bjiu = b[j][i].u; bjiv = b[j][i].v; } else { bjiu = 0.0; bjiv = 0.0; } x[j][i].u = 0.0 + bjiu; x[j][i].v = 0.0 + bjiv; } } /* Test whether we are on the right edge of the global array */ if (xinte == info.mx) { i = info.mx - 1; /* right edge */ for (j=info.ys; j<info.ys+info.ym; j++) { if (B) { bjiu = b[j][i].u; bjiv = b[j][i].v; } else { bjiu = 0.0; bjiv = 0.0; } x[j][i].u = 0.0 + bjiu; x[j][i].v = 0.0 + bjiv; } } for (k=0; k < sweeps; k++) { for (j=info.ys; j<info.ys + info.ym; j++) { for (i=info.xs; i<info.xs + info.xm; i++) { ptconverged = PETSC_FALSE; pfnorm0 = 0.0; fu = 0.0; fv = 0.0; fomega = 0.0; ftemp = 0.0; for (l = 0; l < max_its && !ptconverged; l++) { if (B) { bjiu = b[j][i].u; bjiv = b[j][i].v; bjiomega = b[j][i].omega; bjitemp = b[j][i].temp; } else { bjiu = 0.0; bjiv = 0.0; bjiomega = 0.0; bjitemp = 0.0; } if (i != 0 && i != info.mx - 1 && j != 0 && j != info.my-1) { /* U velocity */ u = x[j][i].u; uxx = (2.0*u - x[j][i-1].u - x[j][i+1].u)*hydhx; uyy = (2.0*u - x[j-1][i].u - x[j+1][i].u)*hxdhy; fu = uxx + uyy - .5*(x[j+1][i].omega-x[j-1][i].omega)*hx - bjiu; dfudu = 2.0*(hydhx + hxdhy); /* V velocity */ u = x[j][i].v; uxx = (2.0*u - x[j][i-1].v - x[j][i+1].v)*hydhx; uyy = (2.0*u - x[j-1][i].v - x[j+1][i].v)*hxdhy; fv = uxx + uyy + .5*(x[j][i+1].omega-x[j][i-1].omega)*hy - bjiv; dfvdv = 2.0*(hydhx + hxdhy); /* convective coefficients for upwinding */ vx = x[j][i].u; avx = PetscAbsScalar(vx); vxp = .5*(vx+avx); vxm = .5*(vx-avx); vy = x[j][i].v; avy = PetscAbsScalar(vy); vyp = .5*(vy+avy); vym = .5*(vy-avy); /* Omega */ u = x[j][i].omega; uxx = (2.0*u - x[j][i-1].omega - x[j][i+1].omega)*hydhx; uyy = (2.0*u - x[j-1][i].omega - x[j+1][i].omega)*hxdhy; fomega = uxx + uyy + (vxp*(u - x[j][i-1].omega) + vxm*(x[j][i+1].omega - u))*hy + (vyp*(u - x[j-1][i].omega) + vym*(x[j+1][i].omega - u))*hx - .5*grashof*(x[j][i+1].temp - x[j][i-1].temp)*hy - bjiomega; /* convective coefficient derivatives */ dfodo = 2.0*(hydhx + hxdhy) + ((vxp - vxm)*hy + (vyp - vym)*hx); if (PetscRealPart(vx) > 0.0) dfodu = (u - x[j][i-1].omega)*hy; else dfodu = (x[j][i+1].omega - u)*hy; if (PetscRealPart(vy) > 0.0) dfodv = (u - x[j-1][i].omega)*hx; else dfodv = (x[j+1][i].omega - u)*hx; /* Temperature */ u = x[j][i].temp; uxx = (2.0*u - x[j][i-1].temp - x[j][i+1].temp)*hydhx; uyy = (2.0*u - x[j-1][i].temp - x[j+1][i].temp)*hxdhy; ftemp = uxx + uyy + prandtl*((vxp*(u - x[j][i-1].temp) + vxm*(x[j][i+1].temp - u))*hy + (vyp*(u - x[j-1][i].temp) + vym*(x[j+1][i].temp - u))*hx) - bjitemp; dftdt = 2.0*(hydhx + hxdhy) + prandtl*((vxp - vxm)*hy + (vyp - vym)*hx); if (PetscRealPart(vx) > 0.0) dftdu = prandtl*(u - x[j][i-1].temp)*hy; else dftdu = prandtl*(x[j][i+1].temp - u)*hy; if (PetscRealPart(vy) > 0.0) dftdv = prandtl*(u - x[j-1][i].temp)*hx; else dftdv = prandtl*(x[j+1][i].temp - u)*hx; /* invert the system: [ dfu / du 0 0 0 ][yu] = [fu] [ 0 dfv / dv 0 0 ][yv] [fv] [ dfo / du dfo / dv dfo / do 0 ][yo] [fo] [ dft / du dft / dv 0 dft / dt ][yt] [ft] by simple back-substitution */ yu = fu / dfudu; yv = fv / dfvdv; yo = (fomega - (dfodu*yu + dfodv*yv)) / dfodo; yt = (ftemp - (dftdu*yu + dftdv*yv)) / dftdt; x[j][i].u = x[j][i].u - yu; x[j][i].v = x[j][i].v - yv; x[j][i].temp = x[j][i].temp - yt; x[j][i].omega = x[j][i].omega - yo; } if (i == 0) { fomega = x[j][i].omega - (x[j][i+1].v - x[j][i].v)*dhx - bjiomega; ftemp = x[j][i].temp - bjitemp; yo = fomega; yt = ftemp; x[j][i].omega = x[j][i].omega - fomega; x[j][i].temp = x[j][i].temp - ftemp; } if (i == info.mx - 1) { fomega = x[j][i].omega - (x[j][i].v - x[j][i-1].v)*dhx - bjiomega; ftemp = x[j][i].temp - (PetscReal)(grashof>0) - bjitemp; yo = fomega; yt = ftemp; x[j][i].omega = x[j][i].omega - fomega; x[j][i].temp = x[j][i].temp - ftemp; } if (j == 0) { fomega = x[j][i].omega + (x[j+1][i].u - x[j][i].u)*dhy - bjiomega; ftemp = x[j][i].temp-x[j+1][i].temp - bjitemp; yo = fomega; yt = ftemp; x[j][i].omega = x[j][i].omega - fomega; x[j][i].temp = x[j][i].temp - ftemp; } if (j == info.my - 1) { fomega = x[j][i].omega + (x[j][i].u - x[j-1][i].u)*dhy - bjiomega; ftemp = x[j][i].temp-x[j-1][i].temp - bjitemp; yo = fomega; yt = ftemp; x[j][i].omega = x[j][i].omega - fomega; x[j][i].temp = x[j][i].temp - ftemp; } tot_its++; pfnorm = PetscRealPart(fu*fu + fv*fv + fomega*fomega + ftemp*ftemp); pfnorm = PetscSqrtReal(pfnorm); pynorm = PetscRealPart(yu*yu + yv*yv + yo*yo + yt*yt); pynorm = PetscSqrtReal(pynorm); pxnorm = PetscRealPart(x[j][i].u*x[j][i].u + x[j][i].v*x[j][i].v + x[j][i].omega*x[j][i].omega + x[j][i].temp*x[j][i].temp); pxnorm = PetscSqrtReal(pxnorm); if (l == 0) pfnorm0 = pfnorm; if (rtol*pfnorm0 >pfnorm || atol > pfnorm || pxnorm*stol > pynorm) ptconverged = PETSC_TRUE; } } } } ierr = DMDAVecRestoreArray(da,localX,&x);CHKERRQ(ierr); if (B) { ierr = DMDAVecRestoreArrayRead(da,localB,&b);CHKERRQ(ierr); } ierr = DMLocalToGlobalBegin(da,localX,INSERT_VALUES,X);CHKERRQ(ierr); ierr = DMLocalToGlobalEnd(da,localX,INSERT_VALUES,X);CHKERRQ(ierr); ierr = PetscLogFlops(tot_its*(84.0 + 41.0 + 26.0));CHKERRQ(ierr); ierr = DMRestoreLocalVector(da,&localX);CHKERRQ(ierr); if (B) { ierr = DMRestoreLocalVector(da,&localB);CHKERRQ(ierr); } PetscFunctionReturn(0); }
PetscErrorCode WASHIFunction(TS ts,PetscReal t,Vec X,Vec Xdot,Vec F,void* ctx) { PetscErrorCode ierr; Wash wash=(Wash)ctx; DM networkdm; Vec localX,localXdot,localF; const PetscInt *cone; PetscInt vfrom,vto,offsetfrom,offsetto,type,varoffset,voffset; PetscInt v,vStart,vEnd,e,eStart,eEnd,pipeoffset; PetscBool ghost; PetscScalar *farr,*vf,*juncx,*juncf; Pipe pipe; PipeField *pipex,*pipexdot,*pipef; DMDALocalInfo info; Junction junction; MPI_Comm comm; PetscMPIInt rank,size; const PetscScalar *xarr,*xdotarr; DMNetworkComponentGenericDataType *nwarr; PetscFunctionBegin; ierr = PetscObjectGetComm((PetscObject)ts,&comm);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); ierr = VecSet(F,0.0);CHKERRQ(ierr); localX = wash->localX; localXdot = wash->localXdot; ierr = TSGetDM(ts,&networkdm);CHKERRQ(ierr); ierr = DMGetLocalVector(networkdm,&localF);CHKERRQ(ierr); ierr = VecSet(localF,0.0);CHKERRQ(ierr); /* update ghost values of locaX and locaXdot */ ierr = DMGlobalToLocalBegin(networkdm,X,INSERT_VALUES,localX);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(networkdm,X,INSERT_VALUES,localX);CHKERRQ(ierr); ierr = DMGlobalToLocalBegin(networkdm,Xdot,INSERT_VALUES,localXdot);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(networkdm,Xdot,INSERT_VALUES,localXdot);CHKERRQ(ierr); ierr = VecGetArrayRead(localX,&xarr);CHKERRQ(ierr); ierr = VecGetArrayRead(localXdot,&xdotarr);CHKERRQ(ierr); ierr = VecGetArray(localF,&farr);CHKERRQ(ierr); /* Initialize localF = localX at non-ghost vertices */ ierr = DMNetworkGetVertexRange(networkdm,&vStart,&vEnd);CHKERRQ(ierr); for (v=vStart; v<vEnd; v++) { ierr = DMNetworkIsGhostVertex(networkdm,v,&ghost);CHKERRQ(ierr); if (!ghost) { ierr = DMNetworkGetVariableOffset(networkdm,v,&varoffset);CHKERRQ(ierr); juncx = (PetscScalar*)(xarr+varoffset); juncf = (PetscScalar*)(farr+varoffset); juncf[0] = juncx[0]; juncf[1] = juncx[1]; } } /* Get component(application) data array */ ierr = DMNetworkGetComponentDataArray(networkdm,&nwarr);CHKERRQ(ierr); /* Edge */ ierr = DMNetworkGetEdgeRange(networkdm,&eStart,&eEnd);CHKERRQ(ierr); for (e=eStart; e<eEnd; e++) { ierr = DMNetworkGetComponentTypeOffset(networkdm,e,0,&type,&pipeoffset);CHKERRQ(ierr); ierr = DMNetworkGetVariableOffset(networkdm,e,&varoffset);CHKERRQ(ierr); pipe = (Pipe)(nwarr + pipeoffset); pipex = (PipeField*)(xarr + varoffset); pipexdot = (PipeField*)(xdotarr + varoffset); pipef = (PipeField*)(farr + varoffset); /* Get boundary values H0 and QL from connected vertices */ ierr = DMNetworkGetConnectedNodes(networkdm,e,&cone);CHKERRQ(ierr); vfrom = cone[0]; /* local ordering */ vto = cone[1]; ierr = DMNetworkGetVariableOffset(networkdm,vfrom,&offsetfrom);CHKERRQ(ierr); ierr = DMNetworkGetVariableOffset(networkdm,vto,&offsetto);CHKERRQ(ierr); if (pipe->boundary.Q0 == PIPE_CHARACTERISTIC) { pipe->boundary.H0 = (xarr+offsetfrom)[1]; /* h_from */ } else { pipe->boundary.Q0 = (xarr+offsetfrom)[0]; /* q_from */ } if (pipe->boundary.HL == PIPE_CHARACTERISTIC) { pipe->boundary.QL = (xarr+offsetto)[0]; /* q_to */ } else { pipe->boundary.HL = (xarr+offsetto)[1]; /* h_to */ } /* Evaluate PipeIFunctionLocal() */ ierr = DMDAGetLocalInfo(pipe->da,&info);CHKERRQ(ierr); ierr = PipeIFunctionLocal(&info, t, pipex, pipexdot, pipef, pipe);CHKERRQ(ierr); /* Set F at vfrom */ vf = (PetscScalar*)(farr+offsetfrom); if (pipe->boundary.Q0 == PIPE_CHARACTERISTIC) { vf[0] -= pipex[0].q; /* q_vfrom - q[0] */ } else { vf[1] -= pipex[0].h; /* h_vfrom - h[0] */ } /* Set F at vto */ vf = (PetscScalar*)(farr+offsetto); if (pipe->boundary.HL == PIPE_CHARACTERISTIC) { vf[1] -= pipex[pipe->nnodes-1].h; /* h_vto - h[last] */ } else { vf[0] -= pipex[pipe->nnodes-1].q; /* q_vto - q[last] */ } } /* Set F at boundary vertices */ for (v=vStart; v<vEnd; v++) { ierr = DMNetworkGetComponentTypeOffset(networkdm,v,0,&type,&voffset);CHKERRQ(ierr); ierr = DMNetworkGetVariableOffset(networkdm,v,&varoffset);CHKERRQ(ierr); junction = (Junction)(nwarr + voffset); juncf = (PetscScalar *)(farr + varoffset); if (junction->isEnd == -1) { juncf[1] -= wash->H0; } else if (junction->isEnd == 1) { juncf[0] -= wash->QL; } } ierr = VecRestoreArrayRead(localX,&xarr);CHKERRQ(ierr); ierr = VecRestoreArrayRead(localXdot,&xdotarr);CHKERRQ(ierr); ierr = VecRestoreArray(localF,&farr);CHKERRQ(ierr); ierr = DMLocalToGlobalBegin(networkdm,localF,ADD_VALUES,F);CHKERRQ(ierr); ierr = DMLocalToGlobalEnd(networkdm,localF,ADD_VALUES,F);CHKERRQ(ierr); ierr = DMRestoreLocalVector(networkdm,&localF);CHKERRQ(ierr); /* ierr = VecView(F,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */ PetscFunctionReturn(0); }
int main(int argc,char **argv) { PetscErrorCode ierr; ObsCtx user; SNES snes; DM da; Vec u, /* solution */ Xu; /* upper bound */ DMDALocalInfo info; PetscReal error1,errorinf; ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,11,11,/* default to 10x10 grid */ PETSC_DECIDE,PETSC_DECIDE, /* number of processors in each dimension */1,/* dof = 1 */1,/* s = 1; stencil extends out one cell */ NULL,NULL,/* do not specify processor decomposition */&da);CHKERRQ(ierr); ierr = DMSetFromOptions(da);CHKERRQ(ierr); ierr = DMSetUp(da);CHKERRQ(ierr); ierr = DMCreateGlobalVector(da,&u);CHKERRQ(ierr); ierr = VecDuplicate(u,&(user.uexact));CHKERRQ(ierr); ierr = VecDuplicate(u,&(user.psi));CHKERRQ(ierr); ierr = DMDASetUniformCoordinates(da,-2.0,2.0,-2.0,2.0,0.0,1.0);CHKERRQ(ierr); ierr = DMSetApplicationContext(da,&user);CHKERRQ(ierr); ierr = FormPsiAndExactSoln(da);CHKERRQ(ierr); ierr = VecSet(u,0.0);CHKERRQ(ierr); ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr); ierr = SNESSetDM(snes,da);CHKERRQ(ierr); ierr = SNESSetApplicationContext(snes,&user);CHKERRQ(ierr); ierr = SNESSetType(snes,SNESVINEWTONRSLS);CHKERRQ(ierr); /* set upper and lower bound constraints for VI */ ierr = VecDuplicate(u,&Xu);CHKERRQ(ierr); ierr = VecSet(Xu,PETSC_INFINITY);CHKERRQ(ierr); ierr = SNESVISetVariableBounds(snes,user.psi,Xu);CHKERRQ(ierr); ierr = VecDestroy(&Xu);CHKERRQ(ierr); ierr = DMDASNESSetFunctionLocal(da,INSERT_VALUES,(PetscErrorCode (*)(DMDALocalInfo*,void*,void*,void*))FormFunctionLocal,&user);CHKERRQ(ierr); ierr = DMDASNESSetJacobianLocal(da,(PetscErrorCode (*)(DMDALocalInfo*,void*,Mat,Mat,void*))FormJacobianLocal,&user);CHKERRQ(ierr); ierr = SNESSetFromOptions(snes);CHKERRQ(ierr); /* report on setup */ ierr = DMDAGetLocalInfo(da,&info); CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"setup done: grid Mx,My = %D,%D with spacing dx,dy = %.4f,%.4f\n", info.mx,info.my,(double)(4.0/(PetscReal)(info.mx-1)),(double)(4.0/(PetscReal)(info.my-1)));CHKERRQ(ierr); /* solve nonlinear system */ ierr = SNESSolve(snes,NULL,u);CHKERRQ(ierr); /* compare to exact */ ierr = VecAXPY(u,-1.0,user.uexact);CHKERRQ(ierr); /* u <- u - uexact */ ierr = VecNorm(u,NORM_1,&error1);CHKERRQ(ierr); error1 /= (PetscReal)info.mx * (PetscReal)info.my; ierr = VecNorm(u,NORM_INFINITY,&errorinf);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"errors: av |u-uexact| = %.3e |u-uexact|_inf = %.3e\n",(double)error1,(double)errorinf);CHKERRQ(ierr); /* Free work space. */ ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&(user.psi));CHKERRQ(ierr); ierr = VecDestroy(&(user.uexact));CHKERRQ(ierr); ierr = SNESDestroy(&snes);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
static PetscErrorCode FormJacobian_All(SNES snes,Vec X,Mat J,Mat B,void *ctx) { User user = (User)ctx; DM dau,dak; DMDALocalInfo infou,infok; PetscScalar *u,*k; PetscErrorCode ierr; Vec Uloc,Kloc; PetscFunctionBeginUser; ierr = DMCompositeGetEntries(user->pack,&dau,&dak);CHKERRQ(ierr); ierr = DMDAGetLocalInfo(dau,&infou);CHKERRQ(ierr); ierr = DMDAGetLocalInfo(dak,&infok);CHKERRQ(ierr); ierr = DMCompositeGetLocalVectors(user->pack,&Uloc,&Kloc);CHKERRQ(ierr); switch (user->ptype) { case 0: ierr = DMGlobalToLocalBegin(dau,X,INSERT_VALUES,Uloc);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd (dau,X,INSERT_VALUES,Uloc);CHKERRQ(ierr); ierr = DMDAVecGetArray(dau,Uloc,&u);CHKERRQ(ierr); ierr = DMDAVecGetArray(dak,user->Kloc,&k);CHKERRQ(ierr); ierr = FormJacobianLocal_U(user,&infou,u,k,B);CHKERRQ(ierr); ierr = DMDAVecRestoreArray(dau,Uloc,&u);CHKERRQ(ierr); ierr = DMDAVecRestoreArray(dak,user->Kloc,&k);CHKERRQ(ierr); break; case 1: ierr = DMGlobalToLocalBegin(dak,X,INSERT_VALUES,Kloc);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd (dak,X,INSERT_VALUES,Kloc);CHKERRQ(ierr); ierr = DMDAVecGetArray(dau,user->Uloc,&u);CHKERRQ(ierr); ierr = DMDAVecGetArray(dak,Kloc,&k);CHKERRQ(ierr); ierr = FormJacobianLocal_K(user,&infok,u,k,B);CHKERRQ(ierr); ierr = DMDAVecRestoreArray(dau,user->Uloc,&u);CHKERRQ(ierr); ierr = DMDAVecRestoreArray(dak,Kloc,&k);CHKERRQ(ierr); break; case 2: { Mat Buu,Buk,Bku,Bkk; IS *is; ierr = DMCompositeScatter(user->pack,X,Uloc,Kloc);CHKERRQ(ierr); ierr = DMDAVecGetArray(dau,Uloc,&u);CHKERRQ(ierr); ierr = DMDAVecGetArray(dak,Kloc,&k);CHKERRQ(ierr); ierr = DMCompositeGetLocalISs(user->pack,&is);CHKERRQ(ierr); ierr = MatGetLocalSubMatrix(B,is[0],is[0],&Buu);CHKERRQ(ierr); ierr = MatGetLocalSubMatrix(B,is[0],is[1],&Buk);CHKERRQ(ierr); ierr = MatGetLocalSubMatrix(B,is[1],is[0],&Bku);CHKERRQ(ierr); ierr = MatGetLocalSubMatrix(B,is[1],is[1],&Bkk);CHKERRQ(ierr); ierr = FormJacobianLocal_U(user,&infou,u,k,Buu);CHKERRQ(ierr); ierr = FormJacobianLocal_UK(user,&infou,&infok,u,k,Buk);CHKERRQ(ierr); ierr = FormJacobianLocal_KU(user,&infou,&infok,u,k,Bku);CHKERRQ(ierr); ierr = FormJacobianLocal_K(user,&infok,u,k,Bkk);CHKERRQ(ierr); ierr = MatRestoreLocalSubMatrix(B,is[0],is[0],&Buu);CHKERRQ(ierr); ierr = MatRestoreLocalSubMatrix(B,is[0],is[1],&Buk);CHKERRQ(ierr); ierr = MatRestoreLocalSubMatrix(B,is[1],is[0],&Bku);CHKERRQ(ierr); ierr = MatRestoreLocalSubMatrix(B,is[1],is[1],&Bkk);CHKERRQ(ierr); ierr = DMDAVecRestoreArray(dau,Uloc,&u);CHKERRQ(ierr); ierr = DMDAVecRestoreArray(dak,Kloc,&k);CHKERRQ(ierr); ierr = ISDestroy(&is[0]);CHKERRQ(ierr); ierr = ISDestroy(&is[1]);CHKERRQ(ierr); ierr = PetscFree(is);CHKERRQ(ierr); } break; } ierr = DMCompositeRestoreLocalVectors(user->pack,&Uloc,&Kloc);CHKERRQ(ierr); ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd (B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); if (J != B) { ierr = MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd (J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); } PetscFunctionReturn(0); }
/* Fills the local vector problem on the subdomain from the global problem. Right now this assumes one subdomain per processor. */ PetscErrorCode DMCreateDomainDecompositionScatters_DA(DM dm,PetscInt nsubdms,DM *subdms,VecScatter **iscat,VecScatter **oscat, VecScatter **lscat) { PetscErrorCode ierr; DMDALocalInfo info,subinfo; DM subdm; MatStencil upper,lower; IS idis,isis,odis,osis,gdis; Vec svec,dvec,slvec; PetscInt xm,ym,zm,xs,ys,zs; PetscInt i; PetscFunctionBegin; /* allocate the arrays of scatters */ if (iscat) {ierr = PetscMalloc1(nsubdms,iscat);CHKERRQ(ierr);} if (oscat) {ierr = PetscMalloc1(nsubdms,oscat);CHKERRQ(ierr);} if (lscat) {ierr = PetscMalloc1(nsubdms,lscat);CHKERRQ(ierr);} ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr); for (i = 0; i < nsubdms; i++) { subdm = subdms[i]; ierr = DMDAGetLocalInfo(subdm,&subinfo);CHKERRQ(ierr); ierr = DMDAGetNonOverlappingRegion(subdm,&xs,&ys,&zs,&xm,&ym,&zm);CHKERRQ(ierr); /* create the global and subdomain index sets for the inner domain */ lower.i = xs; lower.j = ys; lower.k = zs; upper.i = xs+xm; upper.j = ys+ym; upper.k = zs+zm; ierr = DMDACreatePatchIS(dm,&lower,&upper,&idis);CHKERRQ(ierr); ierr = DMDACreatePatchIS(subdm,&lower,&upper,&isis);CHKERRQ(ierr); /* create the global and subdomain index sets for the outer subdomain */ lower.i = subinfo.xs; lower.j = subinfo.ys; lower.k = subinfo.zs; upper.i = subinfo.xs+subinfo.xm; upper.j = subinfo.ys+subinfo.ym; upper.k = subinfo.zs+subinfo.zm; ierr = DMDACreatePatchIS(dm,&lower,&upper,&odis);CHKERRQ(ierr); ierr = DMDACreatePatchIS(subdm,&lower,&upper,&osis);CHKERRQ(ierr); /* global and subdomain ISes for the local indices of the subdomain */ /* todo - make this not loop over at nonperiodic boundaries, which will be more involved */ lower.i = subinfo.gxs; lower.j = subinfo.gys; lower.k = subinfo.gzs; upper.i = subinfo.gxs+subinfo.gxm; upper.j = subinfo.gys+subinfo.gym; upper.k = subinfo.gzs+subinfo.gzm; ierr = DMDACreatePatchIS(dm,&lower,&upper,&gdis);CHKERRQ(ierr); /* form the scatter */ ierr = DMGetGlobalVector(dm,&dvec);CHKERRQ(ierr); ierr = DMGetGlobalVector(subdm,&svec);CHKERRQ(ierr); ierr = DMGetLocalVector(subdm,&slvec);CHKERRQ(ierr); if (iscat) {ierr = VecScatterCreate(dvec,idis,svec,isis,&(*iscat)[i]);CHKERRQ(ierr);} if (oscat) {ierr = VecScatterCreate(dvec,odis,svec,osis,&(*oscat)[i]);CHKERRQ(ierr);} if (lscat) {ierr = VecScatterCreate(dvec,gdis,slvec,NULL,&(*lscat)[i]);CHKERRQ(ierr);} ierr = DMRestoreGlobalVector(dm,&dvec);CHKERRQ(ierr); ierr = DMRestoreGlobalVector(subdm,&svec);CHKERRQ(ierr); ierr = DMRestoreLocalVector(subdm,&slvec);CHKERRQ(ierr); ierr = ISDestroy(&idis);CHKERRQ(ierr); ierr = ISDestroy(&isis);CHKERRQ(ierr); ierr = ISDestroy(&odis);CHKERRQ(ierr); ierr = ISDestroy(&osis);CHKERRQ(ierr); ierr = ISDestroy(&gdis);CHKERRQ(ierr); } PetscFunctionReturn(0); }
PetscErrorCode DMDASubDomainDA_Private(DM dm, PetscInt *nlocal, DM **sdm) { DM *da; PetscInt dim,size,i,j,k,idx; PetscErrorCode ierr; DMDALocalInfo info; PetscInt xsize,ysize,zsize; PetscInt xo,yo,zo; PetscInt xs,ys,zs; PetscInt xm=1,ym=1,zm=1; PetscInt xol,yol,zol; PetscInt m=1,n=1,p=1; PetscInt M,N,P; PetscInt pm,mtmp; PetscFunctionBegin; ierr = DMDAGetLocalInfo(dm,&info);CHKERRQ(ierr); ierr = DMDAGetOverlap(dm,&xol,&yol,&zol);CHKERRQ(ierr); ierr = DMDAGetNumLocalSubDomains(dm,&size);CHKERRQ(ierr); ierr = PetscMalloc1(size,&da);CHKERRQ(ierr); if (nlocal) *nlocal = size; dim = info.dim; M = info.xm; N = info.ym; P = info.zm; if (dim == 1) { m = size; } else if (dim == 2) { m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)N))); while (m > 0) { n = size/m; if (m*n*p == size) break; m--; } } else if (dim == 3) { n = (PetscInt)(0.5 + PetscPowReal(((PetscReal)N*N)*((PetscReal)size)/((PetscReal)P*M),(PetscReal)(1./3.))); if (!n) n = 1; while (n > 0) { pm = size/n; if (n*pm == size) break; n--; } if (!n) n = 1; m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)P*n))); if (!m) m = 1; while (m > 0) { p = size/(m*n); if (m*n*p == size) break; m--; } if (M > P && m < p) {mtmp = m; m = p; p = mtmp;} } zs = info.zs; idx = 0; for (k = 0; k < p; k++) { ys = info.ys; for (j = 0; j < n; j++) { xs = info.xs; for (i = 0; i < m; i++) { if (dim == 1) { xm = M/m + ((M % m) > i); } else if (dim == 2) { xm = M/m + ((M % m) > i); ym = N/n + ((N % n) > j); } else if (dim == 3) { xm = M/m + ((M % m) > i); ym = N/n + ((N % n) > j); zm = P/p + ((P % p) > k); } xsize = xm; ysize = ym; zsize = zm; xo = xs; yo = ys; zo = zs; ierr = DMDACreate(PETSC_COMM_SELF,&(da[idx]));CHKERRQ(ierr); ierr = DMSetOptionsPrefix(da[idx],"sub_");CHKERRQ(ierr); ierr = DMSetDimension(da[idx], info.dim);CHKERRQ(ierr); ierr = DMDASetDof(da[idx], info.dof);CHKERRQ(ierr); ierr = DMDASetStencilType(da[idx],info.st);CHKERRQ(ierr); ierr = DMDASetStencilWidth(da[idx],info.sw);CHKERRQ(ierr); if (info.bx == DM_BOUNDARY_PERIODIC || (xs != 0)) { xsize += xol; xo -= xol; } if (info.by == DM_BOUNDARY_PERIODIC || (ys != 0)) { ysize += yol; yo -= yol; } if (info.bz == DM_BOUNDARY_PERIODIC || (zs != 0)) { zsize += zol; zo -= zol; } if (info.bx == DM_BOUNDARY_PERIODIC || (xs+xm != info.mx)) xsize += xol; if (info.by == DM_BOUNDARY_PERIODIC || (ys+ym != info.my)) ysize += yol; if (info.bz == DM_BOUNDARY_PERIODIC || (zs+zm != info.mz)) zsize += zol; if (info.bx != DM_BOUNDARY_PERIODIC) { if (xo < 0) { xsize += xo; xo = 0; } if (xo+xsize > info.mx-1) { xsize -= xo+xsize - info.mx; } } if (info.by != DM_BOUNDARY_PERIODIC) { if (yo < 0) { ysize += yo; yo = 0; } if (yo+ysize > info.my-1) { ysize -= yo+ysize - info.my; } } if (info.bz != DM_BOUNDARY_PERIODIC) { if (zo < 0) { zsize += zo; zo = 0; } if (zo+zsize > info.mz-1) { zsize -= zo+zsize - info.mz; } } ierr = DMDASetSizes(da[idx], xsize, ysize, zsize);CHKERRQ(ierr); ierr = DMDASetNumProcs(da[idx], 1, 1, 1);CHKERRQ(ierr); ierr = DMDASetBoundaryType(da[idx], DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED, DM_BOUNDARY_GHOSTED);CHKERRQ(ierr); /* set up as a block instead */ ierr = DMSetUp(da[idx]);CHKERRQ(ierr); /* nonoverlapping region */ ierr = DMDASetNonOverlappingRegion(da[idx],xs,ys,zs,xm,ym,zm);CHKERRQ(ierr); /* this alters the behavior of DMDAGetInfo, DMDAGetLocalInfo, DMDAGetCorners, and DMDAGetGhostedCorners and should be used with care */ ierr = DMDASetOffset(da[idx],xo,yo,zo,info.mx,info.my,info.mz);CHKERRQ(ierr); xs += xm; idx++; } ys += ym; } zs += zm; } *sdm = da; PetscFunctionReturn(0); }
/* L_2Error - Integrate the L_2 error of our solution over each face */ PetscErrorCode L_2Error(DM da, Vec fVec, PetscReal *error, AppCtx *user) { DMDALocalInfo info; Vec fLocalVec; Field **f; Field u, uExact, uLocal[4]; PetscScalar hx, hy, hxhy, x, y, phi[3]; PetscInt i, j, q; PetscErrorCode ierr; PetscFunctionBeginUser; ierr = DMDAGetLocalInfo(da, &info);CHKERRQ(ierr); ierr = DMGetLocalVector(da, &fLocalVec);CHKERRQ(ierr); ierr = DMGlobalToLocalBegin(da,fVec, INSERT_VALUES, fLocalVec);CHKERRQ(ierr); ierr = DMGlobalToLocalEnd(da,fVec, INSERT_VALUES, fLocalVec);CHKERRQ(ierr); ierr = DMDAVecGetArray(da, fLocalVec, &f);CHKERRQ(ierr); *error = 0.0; hx = 1.0/(PetscReal)(info.mx-1); hy = 1.0/(PetscReal)(info.my-1); hxhy = hx*hy; for (j = info.ys; j < info.ys+info.ym-1; j++) { for (i = info.xs; i < info.xs+info.xm-1; i++) { uLocal[0] = f[j][i]; uLocal[1] = f[j][i+1]; uLocal[2] = f[j+1][i+1]; uLocal[3] = f[j+1][i]; /* Lower element */ for (q = 0; q < 4; q++) { phi[0] = 1.0 - quadPoints[q*2] - quadPoints[q*2+1]; phi[1] = quadPoints[q*2]; phi[2] = quadPoints[q*2+1]; u.u = uLocal[0].u*phi[0]+ uLocal[1].u*phi[1] + uLocal[3].u*phi[2]; u.v = uLocal[0].v*phi[0]+ uLocal[1].v*phi[1] + uLocal[3].v*phi[2]; u.p = uLocal[0].p*phi[0]+ uLocal[1].p*phi[1] + uLocal[3].p*phi[2]; x = (quadPoints[q*2] + (PetscReal)i)*hx; y = (quadPoints[q*2+1] + (PetscReal)j)*hy; ierr = ExactSolution(PetscAbsScalar(x), PetscAbsScalar(y), &uExact);CHKERRQ(ierr); *error += PetscAbsScalar(hxhy*quadWeights[q]*((u.u - uExact.u)*(u.u - uExact.u) + (u.v - uExact.v)*(u.v - uExact.v) + (u.p - uExact.p)*(u.p - uExact.p))); } /* Upper element */ /* The affine map from the lower to the upper is / x_U \ = / -1 0 \ / x_L \ + / hx \ \ y_U / \ 0 -1 / \ y_L / \ hy / */ for (q = 0; q < 4; q++) { phi[0] = 1.0 - quadPoints[q*2] - quadPoints[q*2+1]; phi[1] = quadPoints[q*2]; phi[2] = quadPoints[q*2+1]; u.u = uLocal[2].u*phi[0]+ uLocal[3].u*phi[1] + uLocal[1].u*phi[2]; u.v = uLocal[2].v*phi[0]+ uLocal[3].v*phi[1] + uLocal[1].v*phi[2]; u.p = uLocal[0].p*phi[0]+ uLocal[1].p*phi[1] + uLocal[3].p*phi[2]; x = (1.0 - quadPoints[q*2] + (PetscReal)i)*hx; y = (1.0 - quadPoints[q*2+1] + (PetscReal)j)*hy; ierr = ExactSolution(PetscAbsScalar(x), PetscAbsScalar(y), &uExact);CHKERRQ(ierr); *error += PetscAbsScalar(hxhy*quadWeights[q]*((u.u - uExact.u)*(u.u - uExact.u) + (u.v - uExact.v)*(u.v - uExact.v) + (u.p - uExact.p)*(u.p - uExact.p))); } } } ierr = DMDAVecRestoreArray(da, fLocalVec, &f);CHKERRQ(ierr); /* ierr = DMLocalToGlobalBegin(da,xLocalVec,ADD_VALUES,xVec);CHKERRQ(ierr); */ /* ierr = DMLocalToGlobalEnd(da,xLocalVec,ADD_VALUES,xVec);CHKERRQ(ierr); */ ierr = DMRestoreLocalVector(da, &fLocalVec);CHKERRQ(ierr); PetscFunctionReturn(0); }