/* Setup for the custom preconditioner */ PetscErrorCode MyPCSetUp(PC pc) { AppCtx *app; PetscErrorCode ierr; DA da; PetscFunctionBegin; ierr = PCShellGetContext(pc,(void**)&app); CHKERRQ(ierr); /* create the linear solver for the Neutron diffusion */ ierr = DMMGCreate(app->comm,1,0,&app->fdmmg); CHKERRQ(ierr); ierr = DMMGSetOptionsPrefix(app->fdmmg,"phi_"); CHKERRQ(ierr); ierr = DMMGSetUser(app->fdmmg,0,app); CHKERRQ(ierr); ierr = DACreate2d(app->comm,DA_NONPERIODIC,DA_STENCIL_STAR,app->nxv,app->nyvf,PETSC_DETERMINE,1,1,1,0,0,&da); CHKERRQ(ierr); ierr = DMMGSetDM(app->fdmmg,(DM)da); CHKERRQ(ierr); ierr = DMMGSetKSP(app->fdmmg,PETSC_NULL,MyFormMatrix); CHKERRQ(ierr); app->dx = DMMGGetRHS(app->fdmmg); app->dy = DMMGGetx(app->fdmmg); ierr = VecDuplicate(app->dy,&app->c); CHKERRQ(ierr); ierr = DADestroy(da); CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **argv) { PetscInt dof = 2,M = 3,N = 3,m = PETSC_DECIDE,n = PETSC_DECIDE; PetscErrorCode ierr; DA da; Vec global,local; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = PetscOptionsGetInt(0,"-dof",&dof,0);CHKERRQ(ierr); /* Create distributed array and get vectors */ ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_BOX, M,N,m,n,dof,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr); ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr); ierr = doit(da,global);CHKERRQ(ierr); ierr = VecView(global,0);CHKERRQ(ierr); /* Free memory */ ierr = VecDestroy(local);CHKERRQ(ierr); ierr = VecDestroy(global);CHKERRQ(ierr); ierr = DADestroy(da);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
void createMG(LocalData* data) { assert(data != NULL); int nlevels = 1; PetscOptionsGetInt(PETSC_NULL, "-nlevels", &nlevels, PETSC_NULL); int coarseSize = 1 + (((data->N) - 1)>>(nlevels - 1)); int rank; MPI_Comm_rank((data->commAll), &rank); if(!rank) { std::cout<<"nlevels = "<<nlevels<<std::endl; std::cout<<"coarseSize = "<<coarseSize<<std::endl; } DMMGCreate(PETSC_COMM_SELF, -nlevels, PETSC_NULL, &(data->mgObj)); DMMGSetOptionsPrefix(data->mgObj, "loc_"); DA da; DACreate2d(PETSC_COMM_SELF, DA_NONPERIODIC, DA_STENCIL_BOX, coarseSize, coarseSize, PETSC_DECIDE, PETSC_DECIDE, (data->dofsPerNode), 1, PETSC_NULL, PETSC_NULL, &da); DMMGSetDM((data->mgObj), (DM)da); DADestroy(da); DMMGSetKSP((data->mgObj), PETSC_NULL, &computeMGmatrix); PetscObjectIncrementTabLevel((PetscObject)(DMMGGetKSP(data->mgObj)), PETSC_NULL, 2); }
int main( int argc, char **args) { PetscErrorCode ierr; ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD, "Start\n"); CHKERRQ(ierr); DA da; ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR, 64,64,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr); int rank; MPI_Comm_rank(PETSC_COMM_WORLD, &rank); printf("[%d] rank\n", rank); Vec p; ierr = DACreateGlobalVector(da,&p); CHKERRQ(ierr); ierr = VecSet( p, 1.); char file[256] = "vec"; sprintf( file, "vec-%d", rank); PetscViewer view; ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_WRITE,&view ); CHKERRQ(ierr); ierr = VecView(p, view); CHKERRQ(ierr); ierr = PetscViewerDestroy(view); CHKERRQ(ierr); ierr = VecDestroy( p ); ierr = DADestroy( da ); ierr = PetscPrintf(PETSC_COMM_WORLD, "End\n"); CHKERRQ(ierr); ierr = PetscFinalize(); CHKERRQ(ierr); }
int main(int argc,char **argv) { Vec u,xy; DA da; PetscErrorCode ierr; PetscInt m = 10, n = 10, dof = 2; PF pf; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_BOX,m,n,PETSC_DECIDE,PETSC_DECIDE,dof,1,0,0,&da);CHKERRQ(ierr); ierr = DASetUniformCoordinates(da,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr); ierr = DACreateGlobalVector(da,&u);CHKERRQ(ierr); ierr = DAGetCoordinates(da,&xy);CHKERRQ(ierr); ierr = DACreatePF(da,&pf);CHKERRQ(ierr); ierr = PFSet(pf,myfunction,0,0,0,0);CHKERRQ(ierr); ierr = PFSetFromOptions(pf);CHKERRQ(ierr); ierr = PFApplyVec(pf,xy,u);CHKERRQ(ierr); ierr = VecView(u,PETSC_VIEWER_DRAW_WORLD);CHKERRQ(ierr); /* Free work space. All PETSc objects should be destroyed when they are no longer needed. */ ierr = VecDestroy(xy);CHKERRQ(ierr); ierr = PFDestroy(pf);CHKERRQ(ierr); ierr = DADestroy(da);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
int main(int argc,char **argv) { PetscErrorCode ierr; PetscViewer viewer; DA da; Vec global,local,global2; PetscMPIInt rank; PetscTruth flg; /* Every PETSc routine should begin with the PetscInitialize() routine. argc, argv - These command line arguments are taken to extract the options supplied to PETSc and options supplied to MPI. help - When PETSc executable is invoked with the option -help, it prints the various options that can be applied at runtime. The user can use the "help" variable place additional help messages in this printout. */ ierr = PetscInitialize(&argc,&argv,(char *)0,help);CHKERRQ(ierr); /* Create a DA and an associated vector */ ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_BOX,100,90,PETSC_DECIDE,PETSC_DECIDE,2,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr); ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr); ierr = VecSet(global,-1.0);CHKERRQ(ierr); ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = VecScale(local,rank+1);CHKERRQ(ierr); ierr = DALocalToGlobal(da,local,ADD_VALUES,global);CHKERRQ(ierr); /* Write output file with PetscViewerHDF5 viewer. */ ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"hdf5output",FILE_MODE_WRITE,&viewer); CHKERRQ(ierr); ierr = VecView(global,viewer);CHKERRQ(ierr); ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr); ierr = VecDuplicate(global,&global2);CHKERRQ(ierr); ierr = VecCopy(global,global2);CHKERRQ(ierr); ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"hdf5output",FILE_MODE_READ,&viewer); CHKERRQ(ierr); ierr = VecLoadIntoVector(viewer,global);CHKERRQ(ierr); ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr); ierr = VecEqual(global,global2,&flg);CHKERRQ(ierr); if (flg) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Vectors are equal\n");CHKERRQ(ierr); } else { ierr = PetscPrintf(PETSC_COMM_WORLD,"Vectors are not equal\n");CHKERRQ(ierr); } /* clean up and exit */ ierr = DADestroy(da);CHKERRQ(ierr); ierr = VecDestroy(local);CHKERRQ(ierr); ierr = VecDestroy(global);CHKERRQ(ierr); ierr = VecDestroy(global2);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
int main(int argc,char **argv) { DMMG *dmmg; /* multilevel grid structure */ AppCtx user; /* user-defined work context */ PetscInt mx,my,its; PetscErrorCode ierr; MPI_Comm comm; SNES snes; DA da2; PetscInitialize(&argc,&argv,(char *)0,help); comm = PETSC_COMM_WORLD; /* Problem parameters (velocity of lid, prandtl, and grashof numbers) */ ierr = PetscOptionsGetReal(PETSC_NULL,"-lidvelocity",&user.lidvelocity,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetReal(PETSC_NULL,"-prandtl",&user.prandtl,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetReal(PETSC_NULL,"-grashof",&user.grashof,PETSC_NULL);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create user context, set problem data, create vector data structures. Also, compute the initial guess. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Setup Physics 2: - Lap(T) + PR*Div([U*T,V*T]) = 0 where U and V are given by the given x.u and x.v - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DACreate2d(comm,DA_NONPERIODIC,DA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0,&da2);CHKERRQ(ierr); ierr = DASetFieldName(da2,0,"temperature");CHKERRQ(ierr); /* Create the solver object and attach the grid/physics info */ ierr = DMMGCreate(comm,1,&user,&dmmg);CHKERRQ(ierr); ierr = DMMGSetDM(dmmg,(DM)da2);CHKERRQ(ierr); ierr = DMMGSetISColoringType(dmmg,IS_COLORING_GLOBAL);CHKERRQ(ierr); ierr = DMMGSetInitialGuess(dmmg,FormInitialGuess);CHKERRQ(ierr); ierr = DMMGSetSNES(dmmg,FormFunction,0);CHKERRQ(ierr); ierr = DMMGSetFromOptions(dmmg);CHKERRQ(ierr); ierr = DAGetInfo(da2,PETSC_NULL,&mx,&my,0,0,0,0,0,0,0,0);CHKERRQ(ierr); user.lidvelocity = 1.0/(mx*my); user.prandtl = 1.0; user.grashof = 1.0; /* Solve the nonlinear system */ ierr = DMMGSolve(dmmg);CHKERRQ(ierr); snes = DMMGGetSNES(dmmg); ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr); ierr = PetscPrintf(comm,"Physics 2: Number of Newton iterations = %D\n\n", its);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Free spaces - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DADestroy(da2);CHKERRQ(ierr); ierr = DMMGDestroy(dmmg);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
void PETSC_STDCALL dacreate2d_(MPI_Comm *comm,DAPeriodicType *wrap,DAStencilType *stencil_type,PetscInt *M,PetscInt *N,PetscInt *m,PetscInt *n,PetscInt *w, PetscInt *s,PetscInt *lx,PetscInt *ly,DA *inra,PetscErrorCode *ierr) { CHKFORTRANNULLINTEGER(lx); CHKFORTRANNULLINTEGER(ly); *ierr = DACreate2d(MPI_Comm_f2c(*(MPI_Fint *)&*comm),*wrap,*stencil_type,*M,*N,*m,*n,*w,*s,lx,ly,inra); }
PetscErrorCode testCreate2D() { int ga; DA da; DALocalInfo info; Vec vec; PetscErrorCode ierr; PetscFunctionBegin; int d1 = 1453, d2 = 1451; ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR, d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr); ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr); ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr); PetscReal **v; ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr); int xe = info.xs+info.xm, ye = info.ys+info.ym; for (int j = info.ys; j < ye; ++j) { for (int i = info.xs; i < xe; ++i) { v[j][i] = 1.*i + d1 * j; } } ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr); PetscPrintf(PETSC_COMM_WORLD,"Updated local portion with DAVec\n"); PetscBarrier(0); { double *da_ptr; VecGetArray(vec, &da_ptr); double *ptr; int low[2],hi[2],ld; NGA_Distribution(ga,GA_Nodeid(),low,hi); NGA_Access(ga,low,hi,&ptr,&ld); printf("[%d] ga:%p\tda:%p\tdiff:%p\n", GA_Nodeid(), ptr, da_ptr, (ptr-da_ptr) ); NGA_Release_update(ga,low,hi); } int lo[2],ld; double val; for (int j = 0; j < d2; ++j) { for (int i = 0; i < d1; ++i) { lo[0] = j; lo[1] = i; NGA_Get(ga,lo,lo,&val,&ld); if( PetscAbs( i + d1*j - val) > .1 ) printf("."); // printf("[%d] (%3.0f,%3.0f)\n", GA_Nodeid(), 1.*i + d1*j, val); } } GA_Print_stats(); ierr = VecDestroy(vec); CHKERRQ(ierr); GA_Destroy(ga); PetscFunctionReturn(0); }
int main(int argc,char **argv) { DMMG *dmmg; /* multilevel grid structure */ PetscErrorCode ierr; DA da; AppCtx app; PC pc; KSP ksp; PetscTruth isshell; PetscViewer v1; PetscInitialize(&argc,&argv,(char *)0,help); PreLoadBegin(PETSC_TRUE,"SetUp"); app.comm = PETSC_COMM_WORLD; app.nxv = 6; app.nyvf = 3; app.nyv = app.nyvf + 2; ierr = PetscOptionsBegin(app.comm,PETSC_NULL,"Options for Grid Sizes",PETSC_NULL); ierr = PetscOptionsInt("-nxv","Grid spacing in X direction",PETSC_NULL,app.nxv,&app.nxv,PETSC_NULL); CHKERRQ(ierr); ierr = PetscOptionsInt("-nyvf","Grid spacing in Y direction of Fuel",PETSC_NULL,app.nyvf,&app.nyvf,PETSC_NULL); CHKERRQ(ierr); ierr = PetscOptionsInt("-nyv","Total Grid spacing in Y direction of",PETSC_NULL,app.nyv,&app.nyv,PETSC_NULL); CHKERRQ(ierr); ierr = PetscOptionsEnd(); ierr = PetscViewerDrawOpen(app.comm,PETSC_NULL,"",-1,-1,-1,-1,&v1); CHKERRQ(ierr); /* Create the DMComposite object to manage the three grids/physics. We use a 1d decomposition along the y direction (since one of the grids is 1d). */ ierr = DMCompositeCreate(app.comm,&app.pack); CHKERRQ(ierr); /* 6 fluid unknowns, 3 ghost points on each end for either periodicity or simply boundary conditions */ ierr = DACreate1d(app.comm,DA_XPERIODIC,app.nxv,6,3,0,&da); CHKERRQ(ierr); ierr = DASetFieldName(da,0,"prss"); CHKERRQ(ierr); ierr = DASetFieldName(da,1,"ergg"); CHKERRQ(ierr); ierr = DASetFieldName(da,2,"ergf"); CHKERRQ(ierr); ierr = DASetFieldName(da,3,"alfg"); CHKERRQ(ierr); ierr = DASetFieldName(da,4,"velg"); CHKERRQ(ierr); ierr = DASetFieldName(da,5,"velf"); CHKERRQ(ierr); ierr = DMCompositeAddDM(app.pack,(DM)da); CHKERRQ(ierr); ierr = DADestroy(da); CHKERRQ(ierr); ierr = DACreate2d(app.comm,DA_YPERIODIC,DA_STENCIL_STAR,app.nxv,app.nyv,PETSC_DETERMINE,1,1,1,0,0,&da); CHKERRQ(ierr); ierr = DASetFieldName(da,0,"Tempature"); CHKERRQ(ierr); ierr = DMCompositeAddDM(app.pack,(DM)da); CHKERRQ(ierr); ierr = DADestroy(da); CHKERRQ(ierr); ierr = DACreate2d(app.comm,DA_XYPERIODIC,DA_STENCIL_STAR,app.nxv,app.nyvf,PETSC_DETERMINE,1,2,1,0,0,&da); CHKERRQ(ierr); ierr = DASetFieldName(da,0,"Phi"); CHKERRQ(ierr); ierr = DASetFieldName(da,1,"Pre"); CHKERRQ(ierr); ierr = DMCompositeAddDM(app.pack,(DM)da); CHKERRQ(ierr); ierr = DADestroy(da); CHKERRQ(ierr); app.pri = 1.0135e+5; app.ugi = 2.5065e+6; app.ufi = 4.1894e+5; app.agi = 1.00e-1; app.vgi = 1.0e-1 ; app.vfi = 1.0e-1; app.prin = 1.0135e+5; app.ugin = 2.5065e+6; app.ufin = 4.1894e+5; app.agin = 1.00e-1; app.vgin = 1.0e-1 ; app.vfin = 1.0e-1; app.prout = 1.0135e+5; app.ugout = 2.5065e+6; app.ufout = 4.1894e+5; app.agout = 3.0e-1; app.twi = 373.15e+0; app.phii = 1.0e+0; app.prei = 1.0e-5; /* Create the solver object and attach the grid/physics info */ ierr = DMMGCreate(app.comm,1,0,&dmmg); CHKERRQ(ierr); ierr = DMMGSetDM(dmmg,(DM)app.pack); CHKERRQ(ierr); ierr = DMMGSetUser(dmmg,0,&app); CHKERRQ(ierr); ierr = DMMGSetISColoringType(dmmg,IS_COLORING_GLOBAL); CHKERRQ(ierr); CHKMEMQ; ierr = DMMGSetInitialGuess(dmmg,FormInitialGuess); CHKERRQ(ierr); ierr = DMMGSetSNES(dmmg,FormFunction,0); CHKERRQ(ierr); ierr = DMMGSetFromOptions(dmmg); CHKERRQ(ierr); /* Supply custom shell preconditioner if requested */ ierr = SNESGetKSP(DMMGGetSNES(dmmg),&ksp); CHKERRQ(ierr); ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr); ierr = PetscTypeCompare((PetscObject)pc,PCSHELL,&isshell); CHKERRQ(ierr); if (isshell) { ierr = PCShellSetContext(pc,&app); CHKERRQ(ierr); ierr = PCShellSetSetUp(pc,MyPCSetUp); CHKERRQ(ierr); ierr = PCShellSetApply(pc,MyPCApply); CHKERRQ(ierr); ierr = PCShellSetDestroy(pc,MyPCDestroy); CHKERRQ(ierr); } /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Solve the nonlinear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ PreLoadStage("Solve"); ierr = DMMGSolve(dmmg); CHKERRQ(ierr); ierr = VecView(DMMGGetx(dmmg),v1); CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Free work space. All PETSc objects should be destroyed when they are no longer needed. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = PetscViewerDestroy(v1); CHKERRQ(ierr); ierr = DMCompositeDestroy(app.pack); CHKERRQ(ierr); ierr = DMMGDestroy(dmmg); CHKERRQ(ierr); PreLoadEnd(); ierr = PetscFinalize(); CHKERRQ(ierr); return 0; }
int main(int argc,char **args) { PetscInt rank,size,npt; PetscScalar dx,dy,cx,cy; PetscErrorCode ierr; Vec x,x0,tempvec, *vinda,*vindb,*vindc; PetscInt i,j,k,n,n2,pmax,puse,Istart,Iend,localsize,niter; PetscScalar **x0array, **aarray,**barray; PetscInt *cacheInt; PetscScalar *cacheScalar; DA myDA; PetscScalar *Mixnorm; PetscInt iter,*iterind,*nind; FILE *fidoutput, *fidtimelog; char fname[50],ftimelog[50]; PetscViewer socketviewer; PetscInt withMatlab, doFFT, doSmoothing; PetscTruth Matlabflag, FFTflag, Smoothingflag; PetscInt timelogcount; MPI_Status status; PetscLogDouble v1,v2,elapsed_time; timelogcount = 0; PetscInitialize(&argc,&args,(char *)0,help); MPI_Comm_size(PETSC_COMM_WORLD,&size); MPI_Comm_rank(PETSC_COMM_WORLD,&rank); ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n"); ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size); ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr); if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;} ierr = PetscOptionsGetInt(PETSC_NULL,"-doFFT",&doFFT,&FFTflag);CHKERRQ(ierr); if (FFTflag == PETSC_FALSE){doFFT = 0;}else{doFFT = 1;} ierr = PetscOptionsGetInt(PETSC_NULL,"-doSmoothing",&doSmoothing,&Smoothingflag);CHKERRQ(ierr); if (Smoothingflag == PETSC_FALSE){doSmoothing = 0;}else{doSmoothing = 1;} if(withMatlab==1){ // Rank 0 connects to socket, use default socket PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer); ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr); // Receive n from Matlab IntReceive(socketviewer, &nind); n = *nind; // Receive iter from Matlab IntReceive(socketviewer, &iterind); iter = *iterind; }else{ ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr); } ///////////////////////////////////////////////////////////////////////////////////// ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n); ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter); Mixnorm = malloc(iter*sizeof(PetscScalar)); dx = 1.0/n; dy = 1.0/n; n2 = (PetscInt)(n*0.5); npt = 5; pmax = 5e5; puse = pmax; PetscInt logmax = 1000; PetscScalar Timelog[logmax]; PetscLogDouble t1,t2; ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*17 ); ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*1); ///////////////////////////////////////////////////////////////////////////////////// // ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr); // ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr); // localsize = Iend-Istart; // ierr = VecDestroy(tempvec);CHKERRQ(ierr); ///////////////////////////////////////////////////////////////////////////////////// if(doSmoothing==1){ ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: Now Do DACreate2d \n\n\n\n" ); ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: %d %d %d\n\n\n\n",n2,n,size); DACreate2d(MPI_COMM_WORLD,DA_XYPERIODIC,DA_STENCIL_BOX,n2,n,1,size,1,2,PETSC_NULL,PETSC_NULL,&myDA); DACreateGlobalVector(myDA,&x0); DAGetCorners(myDA,PETSC_NULL,&Istart,PETSC_NULL,PETSC_NULL,&localsize,PETSC_NULL); Iend = Istart+localsize; }else{ ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr); ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr); localsize = Iend-Istart; ierr = VecDestroy(tempvec);CHKERRQ(ierr); VecCreateMPI(PETSC_COMM_WORLD,localsize*n2,PETSC_DETERMINE ,&x0); } //ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n\n\n\nPETSC: So far so good\n\n\n\n"); VecGetArray2d(x0,n2,localsize,0,0,&x0array); // Create initial vector for(j=0;j<localsize;j++){ for(i=0;i<n2;i++){ cx = (Istart+j+0.5)*dx; x0array[i][j] = cos(2*M_PI*cx); } } VecRestoreArray2d(x0,n2,localsize,0,0,&x0array); ierr = VecDuplicate(x0,&x);CHKERRQ(ierr); ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr); PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n ); vinda = &x0; vindb = &x; sprintf(fname, "mixnorm_%d_%d",n,iter); ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr); ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr); /////////////////////////////////////////////////////////////////////////////////////////////////// // Memory allocation for the iteration scheme // cacheInt = malloc(1*pmax*sizeof(PetscInt)); // cacheScalar = malloc(2*pmax*sizeof(PetscScalar)); cacheInt = malloc(2*pmax*sizeof(PetscInt)); cacheScalar = malloc(2*pmax*sizeof(PetscScalar)); /////////////////////////////////////////////////////////////////////////////////////////////////// // Iteration here! for(niter=0;niter<iter;niter++){ ierr = PetscGetTime(&v1);CHKERRQ(ierr); // BackwardAverage(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend); // BackwardAverageR(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend); BackwardAverageRL(vinda, vindb, cacheInt, cacheScalar, n, npt, pmax, Istart,Iend); vindc = vindb; vindb = vinda; vinda = vindc; // if(doSmoothing==1){Smoothing(vinda, vindb,n, myDA, Istart,Iend);} ierr = PetscGetTime(&v2);CHKERRQ(ierr); //vindc = vindb; //vindb = vinda; //vinda = vindc; ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr); *(Mixnorm+niter) = *(Mixnorm+niter)/n; elapsed_time = v2 - v1; PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 ); PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n" ,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 ); } //////////////////////////////////////////////////////////////////////////////////////////////////// //Change oremtation of vector VecGetArray2d(*vinda,n2,localsize,0,0,&aarray); VecGetArray2d(*vindb,localsize,n2,0,0,&barray); for(j=0;j<localsize;j++){ for(i=0;i<n2;i++){ barray[j][i] = aarray[i][j]; } } VecRestoreArray2d(*vinda,n2,localsize,0,0,&aarray); VecRestoreArray2d(*vindb,localsize,n2,0,0,&barray); vindc = vindb; vindb = vinda; vinda = vindc; //////////////////////////////////////////////////////////////////////////////////////////////////// // FFT part if(doFFT==1){FFT2D(*vinda,*vindb, localsize, n, Istart,Iend, iter,doSmoothing);} //////////////////////////////////////////////////////////////////////////////////////////////////// /* if(rank==0){ sprintf(ftimelog, "timelog_%d_%d",n,iter); fidtimelog = fopen(ftimelog,"w"); for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); } fprintf(fidtimelog,"\n "); for(j = 1;j<size;j++){ MPI_Recv(Timelog,timelogcount,MPI_DOUBLE,j,j,PETSC_COMM_WORLD,&status); for(i=0;i<timelogcount;i++){ fprintf(fidtimelog,"%f ",Timelog[i]); } fprintf(fidtimelog,"\n "); } fclose(fidtimelog); }else{ MPI_Send(Timelog ,timelogcount,MPI_DOUBLE,0,rank,PETSC_COMM_WORLD); } PetscFClose(PETSC_COMM_WORLD,fidoutput); */ /////////////////////////////////////////////////////////////////////////// if(withMatlab==1){ VecView(*vinda,socketviewer); PetscScalarView(iter,Mixnorm,socketviewer); } // free(x0array); free(Mixnorm); free(cacheInt); free(cacheScalar); ierr = VecDestroy(x0);CHKERRQ(ierr); ierr = VecDestroy(x);CHKERRQ(ierr); PetscPrintf(PETSC_COMM_WORLD,"Done!"); ////////////////////////////////////////////////////////////////////////////////////// ierr = PetscFinalize();CHKERRQ(ierr);
int main(int argc,char **argv) { DMMG *dmmg_comp; /* multilevel grid structure */ AppCtx user; /* user-defined work context */ PetscInt mx,my,its,max_its,i; PetscErrorCode ierr; MPI_Comm comm; SNES snes; DA da1,da2; DMComposite pack; DMMG *dmmg1,*dmmg2; PetscTruth SolveSubPhysics=PETSC_FALSE,GaussSeidel=PETSC_TRUE,Jacobi=PETSC_FALSE; Vec X1,X1_local,X2,X2_local; PetscViewer viewer; PetscInitialize(&argc,&argv,(char *)0,help); comm = PETSC_COMM_WORLD; /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create user context, set problem data, create vector data structures. Also, compute the initial guess. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Setup Physics 1: - Lap(U) - Grad_y(Omega) = 0 - Lap(V) + Grad_x(Omega) = 0 - Lap(Omega) + Div([U*Omega,V*Omega]) - GR*Grad_x(T) = 0 where T is given by the given x.temp - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DACreate2d(comm,DA_NONPERIODIC,DA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,0,0,&da1);CHKERRQ(ierr); ierr = DASetFieldName(da1,0,"x-velocity");CHKERRQ(ierr); ierr = DASetFieldName(da1,1,"y-velocity");CHKERRQ(ierr); ierr = DASetFieldName(da1,2,"Omega");CHKERRQ(ierr); /* Create the solver object and attach the grid/physics info */ ierr = DMMGCreate(comm,1,&user,&dmmg1);CHKERRQ(ierr); ierr = DMMGSetDM(dmmg1,(DM)da1);CHKERRQ(ierr); ierr = DMMGSetISColoringType(dmmg1,IS_COLORING_GLOBAL);CHKERRQ(ierr); ierr = DMMGSetInitialGuess(dmmg1,FormInitialGuess1);CHKERRQ(ierr); ierr = DMMGSetSNES(dmmg1,FormFunction1,0);CHKERRQ(ierr); ierr = DMMGSetFromOptions(dmmg1);CHKERRQ(ierr); /* Set problem parameters (velocity of lid, prandtl, and grashof numbers) */ ierr = DAGetInfo(da1,PETSC_NULL,&mx,&my,0,0,0,0,0,0,0,0);CHKERRQ(ierr); user.lidvelocity = 1.0/(mx*my); user.prandtl = 1.0; user.grashof = 1000.0; ierr = PetscOptionsGetReal(PETSC_NULL,"-lidvelocity",&user.lidvelocity,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetReal(PETSC_NULL,"-prandtl",&user.prandtl,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetReal(PETSC_NULL,"-grashof",&user.grashof,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsHasName(PETSC_NULL,"-solvesubphysics",&SolveSubPhysics);CHKERRQ(ierr); ierr = PetscOptionsHasName(PETSC_NULL,"-Jacobi",&Jacobi);CHKERRQ(ierr); if (Jacobi) GaussSeidel=PETSC_FALSE; ierr = PetscPrintf(comm,"grashof: %g, ",user.grashof);CHKERRQ(ierr); if (GaussSeidel){ ierr = PetscPrintf(comm,"use Block Gauss-Seidel\n");CHKERRQ(ierr); } else { ierr = PetscPrintf(comm,"use Block Jacobi\n");CHKERRQ(ierr); } ierr = PetscPrintf(comm,"===========================================\n");CHKERRQ(ierr); /* Solve the nonlinear system 1 */ if (SolveSubPhysics){ ierr = DMMGSolve(dmmg1);CHKERRQ(ierr); snes = DMMGGetSNES(dmmg1); ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr); ierr = PetscPrintf(comm,"Physics 1: Number of Newton iterations = %D\n\n", its);CHKERRQ(ierr); } /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Setup Physics 2: - Lap(T) + PR*Div([U*T,V*T]) = 0 where U and V are given by the given x.u and x.v - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DACreate2d(comm,DA_NONPERIODIC,DA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0,&da2);CHKERRQ(ierr); ierr = DASetFieldName(da2,0,"temperature");CHKERRQ(ierr); /* Create the solver object and attach the grid/physics info */ ierr = DMMGCreate(comm,1,&user,&dmmg2);CHKERRQ(ierr); ierr = DMMGSetDM(dmmg2,(DM)da2);CHKERRQ(ierr); ierr = DMMGSetISColoringType(dmmg2,IS_COLORING_GLOBAL);CHKERRQ(ierr); ierr = DMMGSetInitialGuess(dmmg2,FormInitialGuess2);CHKERRQ(ierr); ierr = DMMGSetSNES(dmmg2,FormFunction2,0);CHKERRQ(ierr); ierr = DMMGSetFromOptions(dmmg2);CHKERRQ(ierr); /* Solve the nonlinear system 2 */ if (SolveSubPhysics){ ierr = DMMGSolve(dmmg2);CHKERRQ(ierr); snes = DMMGGetSNES(dmmg2); ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr); ierr = PetscPrintf(comm,"Physics 2: Number of Newton iterations = %D\n\n", its);CHKERRQ(ierr); } /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Solve system 1 and 2 iteratively - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DACreateLocalVector(da1,&X1_local);CHKERRQ(ierr); ierr = DACreateLocalVector(da2,&X2_local);CHKERRQ(ierr); /* Only 1 snes iteration is allowed for each subphysics */ /* snes = DMMGGetSNES(dmmg1); ierr = SNESSetTolerances(snes,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,1,PETSC_DEFAULT);CHKERRQ(ierr); snes = DMMGGetSNES(dmmg2); ierr = SNESSetTolerances(snes,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,1,PETSC_DEFAULT);CHKERRQ(ierr); */ max_its = 5; ierr = PetscOptionsGetInt(PETSC_NULL,"-mp_max_it",&max_its,PETSC_NULL);CHKERRQ(ierr); user.nsolve = 0; for (i=0; i<max_its; i++){ ierr = PetscPrintf(comm,"\nIterative nsolve %D ...\n", user.nsolve);CHKERRQ(ierr); if (!GaussSeidel){ /* get the ghosted X1_local for Physics 2 */ X1 = DMMGGetx(dmmg1); //Jacobian if (i){ierr = DAVecRestoreArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr);} ierr = DAGlobalToLocalBegin(da1,X1,INSERT_VALUES,X1_local);CHKERRQ(ierr); ierr = DAGlobalToLocalEnd(da1,X1,INSERT_VALUES,X1_local);CHKERRQ(ierr); ierr = DAVecGetArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr); } ierr = DMMGSolve(dmmg1);CHKERRQ(ierr); snes = DMMGGetSNES(dmmg1); ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr); if (GaussSeidel){ /* get the ghosted X1_local for Physics 2 */ X1 = DMMGGetx(dmmg1); if (i){ierr = DAVecRestoreArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr);} ierr = DAGlobalToLocalBegin(da1,X1,INSERT_VALUES,X1_local);CHKERRQ(ierr); ierr = DAGlobalToLocalEnd(da1,X1,INSERT_VALUES,X1_local);CHKERRQ(ierr); ierr = DAVecGetArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr); } ierr = PetscPrintf(comm," Iterative physics 1: Number of Newton iterations = %D\n", its);CHKERRQ(ierr); user.nsolve++; ierr = DMMGSolve(dmmg2);CHKERRQ(ierr); snes = DMMGGetSNES(dmmg2); ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr); /* get the ghosted X2_local for Physics 1 */ X2 = DMMGGetx(dmmg2); if (i){ierr = DAVecRestoreArray(da2,X2_local,(Field2 **)&user.x2);CHKERRQ(ierr);} ierr = DAGlobalToLocalBegin(da2,X2,INSERT_VALUES,X2_local);CHKERRQ(ierr); ierr = DAGlobalToLocalEnd(da2,X2,INSERT_VALUES,X2_local);CHKERRQ(ierr); ierr = DAVecGetArray(da2,X2_local,(Field2 **)&user.x2);CHKERRQ(ierr); ierr = PetscPrintf(comm," Iterative physics 2: Number of Newton iterations = %D\n", its);CHKERRQ(ierr); //user.nsolve++; } ierr = DAVecRestoreArray(da1,X1_local,(Field1 **)&user.x1);CHKERRQ(ierr); ierr = DAVecRestoreArray(da2,X2_local,(Field2 **)&user.x2);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create the DMComposite object to manage the two grids/physics. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = PetscPrintf(comm," \n\n DMComposite iteration......\n");CHKERRQ(ierr); ierr = DMCompositeCreate(comm,&pack);CHKERRQ(ierr); ierr = DMCompositeAddDM(pack,(DM)da1);CHKERRQ(ierr); ierr = DMCompositeAddDM(pack,(DM)da2);CHKERRQ(ierr); /* Create the solver object and attach the grid/physics info */ ierr = DMMGCreate(comm,1,&user,&dmmg_comp);CHKERRQ(ierr); ierr = DMMGSetDM(dmmg_comp,(DM)pack);CHKERRQ(ierr); ierr = DMMGSetISColoringType(dmmg_comp,IS_COLORING_GLOBAL);CHKERRQ(ierr); ierr = DMMGSetInitialGuess(dmmg_comp,FormInitialGuessComp);CHKERRQ(ierr); ierr = DMMGSetSNES(dmmg_comp,FormFunctionComp,0);CHKERRQ(ierr); ierr = DMMGSetFromOptions(dmmg_comp);CHKERRQ(ierr); /* Solve the nonlinear system */ /* ierr = DMMGSolve(dmmg_comp);CHKERRQ(ierr); snes = DMMGGetSNES(dmmg_comp); ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr); ierr = PetscPrintf(comm,"Composite Physics: Number of Newton iterations = %D\n\n", its);CHKERRQ(ierr);*/ /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Free spaces - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DMCompositeDestroy(pack);CHKERRQ(ierr); ierr = DADestroy(da1);CHKERRQ(ierr); ierr = DADestroy(da2);CHKERRQ(ierr); ierr = DMMGDestroy(dmmg_comp);CHKERRQ(ierr); ierr = PetscViewerASCIIOpen(comm,"log.py",&viewer);CHKERRQ(ierr); /* -log_summary */ ierr = PetscLogPrintSummaryToPy(comm,viewer);CHKERRQ(ierr); /* -snes_view */ //snes = DMMGGetSNES(dmmg1);CHKERRQ(ierr); ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr); ierr = DMMGDestroy(dmmg1);CHKERRQ(ierr); ierr = DMMGDestroy(dmmg2);CHKERRQ(ierr); ierr = VecDestroy(X1_local);CHKERRQ(ierr); ierr = VecDestroy(X2_local);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
int main(int argc,char **argv) { PetscMPIInt size,rank; PetscInt M=8,dof=1,stencil_width=1,i,start,end,P=5,N = 6,m=PETSC_DECIDE,n=PETSC_DECIDE,p=PETSC_DECIDE,pt = 0,st = 0; PetscErrorCode ierr; PetscTruth flg2,flg3,flg; DAPeriodicType periodic = DA_NONPERIODIC; DAStencilType stencil_type = DA_STENCIL_STAR; DA da; SDA sda; Vec local,global,local_copy; PetscScalar value,*in,*out; PetscReal norm,work; PetscViewer viewer; char filename[PETSC_MAX_PATH_LEN]; FILE *file; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-P",&P,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-dof",&dof,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-stencil_width",&stencil_width,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-periodic",&pt,PETSC_NULL);CHKERRQ(ierr); periodic = (DAPeriodicType) pt; ierr = PetscOptionsGetInt(PETSC_NULL,"-stencil_type",&st,PETSC_NULL);CHKERRQ(ierr); stencil_type = (DAStencilType) st; ierr = PetscOptionsHasName(PETSC_NULL,"-1d",&flg2);CHKERRQ(ierr); ierr = PetscOptionsHasName(PETSC_NULL,"-2d",&flg2);CHKERRQ(ierr); ierr = PetscOptionsHasName(PETSC_NULL,"-3d",&flg3);CHKERRQ(ierr); if (flg2) { ierr = DACreate2d(PETSC_COMM_WORLD,periodic,stencil_type,M,N,m,n,dof,stencil_width,0,0,&da);CHKERRQ(ierr); ierr = SDACreate2d(PETSC_COMM_WORLD,periodic,stencil_type,M,N,m,n,dof,stencil_width,0,0,&sda);CHKERRQ(ierr); } else if (flg3) { ierr = DACreate3d(PETSC_COMM_WORLD,periodic,stencil_type,M,N,P,m,n,p,dof,stencil_width,0,0,0,&da);CHKERRQ(ierr); ierr = SDACreate3d(PETSC_COMM_WORLD,periodic,stencil_type,M,N,P,m,n,p,dof,stencil_width,0,0,0,&sda);CHKERRQ(ierr); } else { ierr = DACreate1d(PETSC_COMM_WORLD,periodic,M,dof,stencil_width,PETSC_NULL,&da);CHKERRQ(ierr); ierr = SDACreate1d(PETSC_COMM_WORLD,periodic,M,dof,stencil_width,PETSC_NULL,&sda);CHKERRQ(ierr); } ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr); ierr = VecDuplicate(local,&local_copy);CHKERRQ(ierr); /* zero out vectors so that ghostpoints are zero */ value = 0; ierr = VecSet(local,value);CHKERRQ(ierr); ierr = VecSet(local_copy,value);CHKERRQ(ierr); ierr = VecGetOwnershipRange(global,&start,&end);CHKERRQ(ierr); for (i=start; i<end; i++) { value = i + 1; ierr = VecSetValues(global,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr); } ierr = VecAssemblyBegin(global);CHKERRQ(ierr); ierr = VecAssemblyEnd(global);CHKERRQ(ierr); ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetTruth(PETSC_NULL,"-same_array",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) { /* test the case where the input and output array is the same */ ierr = VecCopy(local,local_copy);CHKERRQ(ierr); ierr = VecGetArray(local_copy,&in);CHKERRQ(ierr); ierr = VecRestoreArray(local_copy,PETSC_NULL);CHKERRQ(ierr); ierr = SDALocalToLocalBegin(sda,in,INSERT_VALUES,in);CHKERRQ(ierr); ierr = SDALocalToLocalEnd(sda,in,INSERT_VALUES,in);CHKERRQ(ierr); } else { ierr = VecGetArray(local,&out);CHKERRQ(ierr); ierr = VecRestoreArray(local,PETSC_NULL);CHKERRQ(ierr); ierr = VecGetArray(local_copy,&in);CHKERRQ(ierr); ierr = VecRestoreArray(local_copy,PETSC_NULL);CHKERRQ(ierr); ierr = SDALocalToLocalBegin(sda,out,INSERT_VALUES,in);CHKERRQ(ierr); ierr = SDALocalToLocalEnd(sda,out,INSERT_VALUES,in);CHKERRQ(ierr); } flg = PETSC_FALSE; ierr = PetscOptionsGetTruth(PETSC_NULL,"-save",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) { ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); sprintf(filename,"local.%d",rank); ierr = PetscViewerASCIIOpen(PETSC_COMM_SELF,filename,&viewer);CHKERRQ(ierr); ierr = PetscViewerASCIIGetPointer(viewer,&file);CHKERRQ(ierr); ierr = VecView(local,viewer);CHKERRQ(ierr); fprintf(file,"Vector with correct ghost points\n"); ierr = VecView(local_copy,viewer);CHKERRQ(ierr); ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr); } ierr = VecAXPY(local_copy,-1.0,local);CHKERRQ(ierr); ierr = VecNorm(local_copy,NORM_MAX,&work);CHKERRQ(ierr); ierr = MPI_Allreduce(&work,&norm,1,MPIU_REAL,MPI_MAX,PETSC_COMM_WORLD);CHKERRQ(ierr); if (norm != 0.0) { ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of difference %G should be zero\n",norm);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD," Number of processors %d\n",size);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD," M,N,P,dof %D %D %D %D\n",M,N,P,dof);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD," stencil_width %D stencil_type %d periodic %d\n",stencil_width,(int)stencil_type,(int)periodic);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD," dimension %d\n",1 + (int) flg2 + (int) flg3);CHKERRQ(ierr); } ierr = DADestroy(da);CHKERRQ(ierr); ierr = SDADestroy(sda);CHKERRQ(ierr); ierr = VecDestroy(local_copy);CHKERRQ(ierr); ierr = VecDestroy(local);CHKERRQ(ierr); ierr = VecDestroy(global);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
int main(int argc,char **argv) /*-----------------------------------------------------------------------*/ { DMMG *dmmg; /* multilevel grid structure */ AppCtx *user; /* user-defined work context */ Parameter *param; GridInfo grid; int ierr,result; MPI_Comm comm; DA da; PetscInitialize(&argc,&argv,(char *)0,help); comm = PETSC_COMM_WORLD; /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Set up the problem parameters. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = PetscMalloc(sizeof(AppCtx),&user); CHKERRQ(ierr); ierr = PetscBagCreate(comm,sizeof(Parameter),&(user->bag)); CHKERRQ(ierr); user->grid = &grid; ierr = SetParams(user); CHKERRQ(ierr); ierr = ReportParams(user); CHKERRQ(ierr); ierr = PetscBagGetData(user->bag,(void**)¶m); CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create distributed array multigrid object (DMMG) to manage parallel grid and vectors for principal unknowns (x) and governing residuals (f) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DMMGCreate(comm,grid.mglevels,user,&dmmg); CHKERRQ(ierr); ierr = DACreate2d(comm,grid.periodic,grid.stencil,grid.ni,grid.nj,PETSC_DECIDE,PETSC_DECIDE,grid.dof,grid.stencil_width,0,0,&da); CHKERRQ(ierr); ierr = DMMGSetDM(dmmg,(DM)da); CHKERRQ(ierr); ierr = DADestroy(da); CHKERRQ(ierr); ierr = DAGetInfo(da,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,&(param->pi),&(param->pj),PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL); CHKERRQ(ierr); REG_INTG(user->bag,¶m->pi,param->pi ,"procs_x","<DO NOT SET> Processors in the x-direction"); REG_INTG(user->bag,¶m->pj,param->pj ,"procs_y","<DO NOT SET> Processors in the y-direction"); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create user context, set problem data, create vector data structures. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DAGetGlobalVector(da, &(user->Xold)); CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Initialize and solve the nonlinear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = Initialize(dmmg); CHKERRQ(ierr); ierr = DoSolve(dmmg); CHKERRQ(ierr); if (param->verify) result = param->verify_result; /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Free work space. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = DARestoreGlobalVector(da, &(user->Xold)); CHKERRQ(ierr); ierr = PetscBagDestroy(user->bag); CHKERRQ(ierr); ierr = PetscFree(user); CHKERRQ(ierr); ierr = DMMGDestroy(dmmg); CHKERRQ(ierr); ierr = PetscFinalize(); CHKERRQ(ierr); return result; }
int main(int argc,char **argv) { PetscErrorCode ierr; PetscInt its,n,Nx=PETSC_DECIDE,Ny=PETSC_DECIDE,nlocal,i; PetscMPIInt size; PC pc; PetscInt mx,my; Mat A; GridCtx fine_ctx; KSP ksp; PetscTruth flg; PetscInitialize(&argc,&argv,PETSC_NULL,help); /* set up discretization matrix for fine grid */ /* ML requires input of fine-grid matrix. It determines nlevels. */ fine_ctx.mx = 9; fine_ctx.my = 9; ierr = PetscOptionsGetInt(PETSC_NULL,"-mx",&mx,&flg);CHKERRQ(ierr); if (flg) fine_ctx.mx = mx; ierr = PetscOptionsGetInt(PETSC_NULL,"-my",&my,&flg);CHKERRQ(ierr); if (flg) fine_ctx.my = my; ierr = PetscPrintf(PETSC_COMM_WORLD,"Fine grid size %D by %D\n",fine_ctx.mx,fine_ctx.my);CHKERRQ(ierr); n = fine_ctx.mx*fine_ctx.my; MPI_Comm_size(PETSC_COMM_WORLD,&size); ierr = PetscOptionsGetInt(PETSC_NULL,"-Nx",&Nx,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-Ny",&Ny,PETSC_NULL);CHKERRQ(ierr); ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,fine_ctx.mx, fine_ctx.my,Nx,Ny,1,1,PETSC_NULL,PETSC_NULL,&fine_ctx.da);CHKERRQ(ierr); ierr = DACreateGlobalVector(fine_ctx.da,&fine_ctx.x);CHKERRQ(ierr); ierr = VecDuplicate(fine_ctx.x,&fine_ctx.b);CHKERRQ(ierr); ierr = VecGetLocalSize(fine_ctx.x,&nlocal);CHKERRQ(ierr); ierr = DACreateLocalVector(fine_ctx.da,&fine_ctx.localX);CHKERRQ(ierr); ierr = VecDuplicate(fine_ctx.localX,&fine_ctx.localF);CHKERRQ(ierr); ierr = MatCreateMPIAIJ(PETSC_COMM_WORLD,nlocal,nlocal,n,n,5,PETSC_NULL,3,PETSC_NULL,&A);CHKERRQ(ierr); ierr = FormJacobian_Grid(&fine_ctx,&A);CHKERRQ(ierr); /* create linear solver */ ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr); ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr); ierr = PCSetType(pc,PCML);CHKERRQ(ierr); /* set options, then solve system */ ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* calls PCSetFromOptions_MG/ML */ for (i=0; i<3; i++){ if (i<2){ /* test DIFFERENT_NONZERO_PATTERN */ /* set values for rhs vector */ ierr = VecSet(fine_ctx.b,i+1.0);CHKERRQ(ierr); /* modify A */ ierr = MatShift(A,1.0);CHKERRQ(ierr); ierr = MatScale(A,2.0);CHKERRQ(ierr); ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr); } else { /* test SAME_NONZERO_PATTERN */ ierr = KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr); } ierr = KSPSolve(ksp,fine_ctx.b,fine_ctx.x);CHKERRQ(ierr); ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Number of iterations = %D\n",its);CHKERRQ(ierr); } /* free data structures */ ierr = VecDestroy(fine_ctx.x);CHKERRQ(ierr); ierr = VecDestroy(fine_ctx.b);CHKERRQ(ierr); ierr = DADestroy(fine_ctx.da);CHKERRQ(ierr); ierr = VecDestroy(fine_ctx.localX);CHKERRQ(ierr); ierr = VecDestroy(fine_ctx.localF);CHKERRQ(ierr); ierr = MatDestroy(A);CHKERRQ(ierr); ierr = KSPDestroy(ksp);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
int main(int argc,char **argv) { PetscMPIInt rank; PetscInt M = -10,N = -8; PetscErrorCode ierr; PetscTruth flg = PETSC_FALSE; DA da; PetscViewer viewer; Vec local,global; PetscScalar value; DAPeriodicType ptype = DA_NONPERIODIC; DAStencilType stype = DA_STENCIL_BOX; #if defined(PETSC_HAVE_MATLAB_ENGINE) PetscViewer mviewer; #endif ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,300,300,&viewer);CHKERRQ(ierr); #if defined(PETSC_HAVE_MATLAB_ENGINE) ierr = PetscViewerMatlabOpen(PETSC_COMM_WORLD,"tmp.mat",FILE_MODE_WRITE,&mviewer);CHKERRQ(ierr); #endif ierr = PetscOptionsGetTruth(PETSC_NULL,"-star_stencil",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) stype = DA_STENCIL_STAR; /* Create distributed array and get vectors */ ierr = DACreate2d(PETSC_COMM_WORLD,ptype,stype,M,N,PETSC_DECIDE,PETSC_DECIDE,1,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr); ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = DACreateLocalVector(da,&local);CHKERRQ(ierr); value = -3.0; ierr = VecSet(global,value);CHKERRQ(ierr); ierr = DAGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = DAGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); value = rank+1; ierr = VecScale(local,value);CHKERRQ(ierr); ierr = DALocalToGlobal(da,local,ADD_VALUES,global);CHKERRQ(ierr); flg = PETSC_FALSE; ierr = PetscOptionsGetTruth(PETSC_NULL, "-view_global", &flg,PETSC_NULL);CHKERRQ(ierr); if (flg) { /* view global vector in natural ordering */ ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } ierr = DAView(da,viewer);CHKERRQ(ierr); ierr = VecView(global,viewer);CHKERRQ(ierr); #if defined(PETSC_HAVE_MATLAB_ENGINE) ierr = DAView(da,mviewer);CHKERRQ(ierr); ierr = VecView(global,mviewer);CHKERRQ(ierr); #endif /* Free memory */ #if defined(PETSC_HAVE_MATLAB_ENGINE) ierr = PetscViewerDestroy(mviewer);CHKERRQ(ierr); #endif ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr); ierr = VecDestroy(local);CHKERRQ(ierr); ierr = VecDestroy(global);CHKERRQ(ierr); ierr = DADestroy(da);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
int main(int argc,char **argv) { PetscInt i,j,M = 10,N = 8,m = PETSC_DECIDE,n = PETSC_DECIDE; PetscMPIInt rank; PetscErrorCode ierr; PetscTruth flg = PETSC_FALSE; DA da; PetscViewer viewer; Vec localall,global; PetscScalar value,*vlocal; DAPeriodicType ptype = DA_NONPERIODIC; DAStencilType stype = DA_STENCIL_BOX; VecScatter tolocalall,fromlocalall; PetscInt start,end; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,300,300,&viewer);CHKERRQ(ierr); /* Read options */ ierr = PetscOptionsGetInt(PETSC_NULL,"-M",&M,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-N",&N,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr); ierr = PetscOptionsGetTruth(PETSC_NULL,"-star_stencil",&flg,PETSC_NULL);CHKERRQ(ierr); if (flg) stype = DA_STENCIL_STAR; /* Create distributed array and get vectors */ ierr = DACreate2d(PETSC_COMM_WORLD,ptype,stype, M,N,m,n,1,1,PETSC_NULL,PETSC_NULL,&da);CHKERRQ(ierr); ierr = DACreateGlobalVector(da,&global);CHKERRQ(ierr); ierr = VecCreateSeq(PETSC_COMM_SELF,M*N,&localall);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = VecGetOwnershipRange(global,&start,&end);CHKERRQ(ierr); for (i=start; i<end; i++) { value = 5.0*rank; ierr = VecSetValues(global,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr); } ierr = VecView(global,viewer);CHKERRQ(ierr); /* Create Scatter from global DA parallel vector to local vector that contains all entries */ ierr = DAGlobalToNaturalAllCreate(da,&tolocalall);CHKERRQ(ierr); ierr = DANaturalAllToGlobalCreate(da,&fromlocalall);CHKERRQ(ierr); ierr = VecScatterBegin(tolocalall,global,localall,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(tolocalall,global,localall,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecGetArray(localall,&vlocal);CHKERRQ(ierr); for (j=0; j<N; j++) { for (i=0; i<M; i++) { *vlocal++ += i + j*M; } } ierr = VecRestoreArray(localall,&vlocal);CHKERRQ(ierr); /* scatter back to global vector */ ierr = VecScatterBegin(fromlocalall,localall,global,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(fromlocalall,localall,global,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecView(global,viewer);CHKERRQ(ierr); /* Free memory */ ierr = VecScatterDestroy(tolocalall);CHKERRQ(ierr); ierr = VecScatterDestroy(fromlocalall);CHKERRQ(ierr); ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr); ierr = VecDestroy(localall);CHKERRQ(ierr); ierr = VecDestroy(global);CHKERRQ(ierr); ierr = DADestroy(da);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
PetscErrorCode vizGA2DA() { PetscErrorCode ierr; int rank; MPI_Comm_rank(PETSC_COMM_WORLD,&rank); int d1 = 40, d2 = 50; DA da; Vec vec; const PetscInt *lx, *ly, *lz; PetscInt m,n,p; DALocalInfo info; ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR, d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr); ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr); ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr); ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr); ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr); /**/ ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr); for (int i = 0; i < m; ++i) { PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]); } for (int i = 0; i < n; ++i) { PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]); } /**/ int ga = GA_Create_handle(); int ndim = 2; int dims[2] = {d2,d1}; GA_Set_data(ga,2,dims,MT_DBL); int *map; PetscMalloc( sizeof(int)*(m+n), &map); map[0] = 0; for( int i = 1; i < n; i++ ) { map[i] = ly[i-1] + map[i-1]; } map[n] = 0; for( int i = n+1; i < m+n; i++ ) { map[i] = lx[i-n-1] + map[i-1]; } /* correct ordering, but nodeid's dont line up with mpi rank for petsc's da * DA: +---+---+ GA: +---+---+ * +-2-+-3-+ +-1-+-3-+ * +---+---+ +---+---+ * +-0-+-1-+ +-0-+-2-+ * +---+---+ +---+---+ int *map; PetscMalloc( sizeof(int)*(m+n), &map); map[0] = 0; for( int i = 1; i < m; i++ ) { map[i] = lx[i] + map[i-1]; } map[m] = 0; for( int i = m+1; i < m+n; i++ ) { map[i] = ly[i-m] + map[i-1]; } */ int block[2] = {n,m}; GA_Set_irreg_distr(ga,map,block); ierr = GA_Allocate( ga ); if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr); if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr); if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1); GA_Print_distribution(ga); int lo[2], hi[2]; NGA_Distribution(ga,rank,lo,hi); if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 || lo[0] != info.ys || hi[0] != info.ys+info.ym-1 ) { PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n", rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1); } PetscBarrier(0); PetscSynchronizedFlush(PETSC_COMM_WORLD); AO ao; DAGetAO(da,&ao); if( rank == 0 ) { int *idx, len = d1*d2; PetscReal *val; PetscMalloc(sizeof(PetscReal)*len, &val); PetscMalloc(sizeof(int)*len, &idx); for (int j = 0; j < d2; ++j) { for (int i = 0; i < d1; ++i) { idx[i + d1*j] = i + d1*j; val[i + d1*j] = i + d1*j; } } AOApplicationToPetsc(ao,len,idx); VecSetValues(vec,len,idx,val,INSERT_VALUES); int a[2], b[2],ld[1]={0}; double c = 0; for (int j = 0; j < d2; ++j) { for (int i = 0; i < d1; ++i) { a[0] = j; a[1] = i; // printf("%5.0f ",c); NGA_Put(ga,a,a,&c,ld); c++; } } } // GA_Print(ga); VecAssemblyBegin(vec); VecAssemblyEnd(vec); int ld; double *ptr; NGA_Access(ga,lo,hi,&ptr,&ld); PetscReal **d; int c=0; ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr); for (int j = info.ys; j < info.ys+info.ym; ++j) { for (int i = info.xs; i < info.xs+info.xm; ++i) { if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] ) GA_Error("DA array is not equal to GA array",1); // printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]); c++; } } ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr); c=0; PetscReal *v; int start, end; VecGetOwnershipRange(vec, &start, &end); VecGetArray( vec, &v ); for( int i = start; i < end; i++) { // printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") ); } VecRestoreArray( vec, &v ); NGA_Release_update(ga,lo,hi); Vec gada; VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada); VecView(gada,PETSC_VIEWER_STDOUT_SELF); GA_Destroy(ga); ierr = VecDestroy(vec); CHKERRQ(ierr); ierr = DADestroy(da); CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **args) { PetscErrorCode ierr; PetscInitialize(&argc,&args,(char *)0,help); PetscInt m = 10; /* default number of rows and columns IN GRID, but matrix is (m^2) x (m^2) */ ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr); MPI_Comm com = PETSC_COMM_WORLD; PetscMPIInt rank, size; ierr = MPI_Comm_rank(com, &rank); CHKERRQ(ierr); ierr = MPI_Comm_size(com, &size); CHKERRQ(ierr); /* create m x m two-dimensional grid for periodic boundary condition problem */ DA da2; PetscInt dof=1, stencilwidth=1; ierr = DACreate2d(com, DA_XYPERIODIC, DA_STENCIL_STAR, m,m,PETSC_DECIDE,PETSC_DECIDE, dof,stencilwidth,PETSC_NULL,PETSC_NULL,&da2); CHKERRQ(ierr); /* get da2-managed Vecs */ Vec x,b,u; ierr = DACreateGlobalVector(da2,&x); CHKERRQ(ierr); ierr = VecDuplicate(x,&b); CHKERRQ(ierr); ierr = VecDuplicate(x,&u); CHKERRQ(ierr); Mat A; ierr = DAGetMatrix(da2, MATMPIAIJ, &A); CHKERRQ(ierr); /* alternative call below is not quite same as result from DAGetMatrix(), because of nonzero allocation; the Mat ownership ranges are same */ /* ierr = MatCreateMPIAIJ(com, mlocal, mlocal, m*m, m*m, 5, PETSC_NULL, 4, PETSC_NULL, &A); CHKERRQ(ierr) */ ierr = MatSetFromOptions(A);CHKERRQ(ierr); /* report on ownership range */ PetscInt rstart,rend,mlocal; ierr = VecGetOwnershipRange(x,&rstart,&rend);CHKERRQ(ierr); ierr = VecGetLocalSize(x,&mlocal);CHKERRQ(ierr); PetscInt A_rstart,A_rend; ierr = MatGetOwnershipRange(A,&A_rstart,&A_rend);CHKERRQ(ierr); if ((rstart != A_rstart) || (rend != A_rend)) { ierr = PetscPrintf(com, "Vec and Mat ownership ranges different!!! ending ...\n"); CHKERRQ(ierr); PetscEnd(); } else { ierr = PetscSynchronizedPrintf(com, "rank=%d has Vec and Mat ownership: mlocal=%d, rstart=%d, rend=%d\n", rank,mlocal,rstart,rend); CHKERRQ(ierr); } PetscSynchronizedFlush(com); /* get local part of grid */ PetscInt xm,ym,xs,ys; DAGetCorners(da2,&xs,&ys,0,&xm,&ym,0); /* report on local part of grid */ ierr = PetscSynchronizedPrintf(com, "rank=%d has da2-managed-Vec local ranges: xs=%d, xm=%d, ys=%d, ym=%d\n", rank,xs,xm,ys,ym); CHKERRQ(ierr); PetscSynchronizedFlush(com); /* set up linear system */ PetscScalar **barr, **uarr; /* RHS and exact soln, resp. */ DAVecGetArray(da2, b, &barr); DAVecGetArray(da2, u, &uarr); PetscScalar dx = 1.0/(double)m, dy = dx, pi = 3.14159265358979; PetscScalar xi,yj; PetscInt diag=0,north=1,east=2,south=3,west=4; PetscScalar vals[5] = {-4.0 + dx * dx, 1.0, 1.0, 1.0, 1.0}; MatStencil row, col[5]; /* these are not "stencils" at all, but local grid to global indices helpers */ PetscInt i,j,num; for (j=ys; j<ys+ym; j++) { for(i=xs; i<xs+xm; i++) { /* entries of matrix A */ row.i = i; row.j = j; row.c = 0; /* dof = 1 so first component; note row.k is for 3d DAs */ for (num=0; num<5; num++) col[num].c = 0; /* set diag first, then go through stencil neighbors */ col[diag].i = i; col[diag].j = j; col[north].i = i; col[north].j = j+1; col[east].i = i+1; col[east].j = j; col[south].i = i; col[south].j = j-1; col[west].i = i-1; col[west].j = j; ierr = MatSetValuesStencil(A,1,&row,5,col,vals,INSERT_VALUES); CHKERRQ(ierr); /* entries of vectors: exact solution u and right-hand-side b */ xi = (double)i * dx; yj = (double)j * dy; uarr[j][i] = sin(2.0 * pi * xi) * cos(4.0 * pi * yj); barr[j][i] = (1.0 - 20.0 * pi * pi) * uarr[j][i]; barr[j][i] *= dx * dx; } } DAVecRestoreArray(da2, b, &barr); DAVecRestoreArray(da2, u, &uarr); ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr); ierr = VecAssemblyBegin(b); CHKERRQ(ierr); ierr = VecAssemblyEnd(b); CHKERRQ(ierr); ierr = VecAssemblyBegin(u); CHKERRQ(ierr); ierr = VecAssemblyEnd(u); CHKERRQ(ierr); /* uncomment for dense view; -mat_view default format is good enough for most purposes PetscViewer viewer; PetscViewerCreate(com, &viewer); PetscViewerSetType(viewer, PETSC_VIEWER_ASCII); PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_DENSE); MatView(A,viewer); PetscViewerDestroy(viewer); */ /* setup solver context now that Mat and Vec are assembled */ KSP ksp; ierr = KSPCreate(com,&ksp);CHKERRQ(ierr); /* Set "operators". Here the matrix that defines the linear system also serves as the preconditioning matrix. But we do not assert a relationship between their nonzero patterns.(???) */ ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr); /* Following is optional; parameters could be set at runtime. */ ierr = KSPSetTolerances(ksp,1.e-7,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr); /* Set runtime options, e.g., -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol> */ ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* Solve linear system */ ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr); /* Compute and report the error (and the iteration count and reason). */ PetscScalar norminf, normtwo, neg_one=-1.0; PetscInt its; KSPConvergedReason reason; ierr = VecAXPY(x,neg_one,u);CHKERRQ(ierr); // x = x - u ierr = VecNorm(x,NORM_INFINITY,&norminf);CHKERRQ(ierr); ierr = VecNorm(x,NORM_2,&normtwo);CHKERRQ(ierr); // discrete norm normtwo *= dx * dy; // integral norm ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr); ierr = KSPGetConvergedReason(ksp,&reason); CHKERRQ(ierr); ierr = PetscPrintf(com, "Error norms ||err||_inf = %.3e, ||err||_2 = %.3e;\n" "Iterations = %d; Reason = %d\n", norminf, normtwo, its, (int) reason);CHKERRQ(ierr); /* destroy */ ierr = KSPDestroy(ksp);CHKERRQ(ierr); ierr = MatDestroy(A);CHKERRQ(ierr); ierr = VecDestroy(x);CHKERRQ(ierr); ierr = VecDestroy(u);CHKERRQ(ierr); ierr = VecDestroy(b);CHKERRQ(ierr); ierr = DADestroy(da2);CHKERRQ(ierr); /* Always call PetscFinalize() before exiting a program. */ ierr = PetscFinalize();CHKERRQ(ierr); return 0; }