/*@C
PetscHMPISpawn - Initialize additional processes to be used as "worker" processes. This is not generally
called by users. One should use -hmpi_spawn_size <n> to indicate that you wish to have n-1 new MPI
processes spawned for each current process.
Not Collective (could make collective on MPI_COMM_WORLD, generate one huge comm and then split it up)
Input Parameter:
. nodesize - size of each compute node that will share processors
Options Database:
. -hmpi_spawn_size nodesize
Notes: This is only supported on systems with an MPI 2 implementation that includes the MPI_Comm_Spawn() routine.
$ Comparison of two approaches for HMPI usage (MPI started with N processes)
$
$ -hmpi_spawn_size <n> requires MPI 2, results in n*N total processes with N directly used by application code
$ and n-1 worker processes (used by PETSc) for each application node.
$ You MUST launch MPI so that only ONE MPI process is created for each hardware node.
$
$ -hmpi_merge_size <n> results in N total processes, N/n used by the application code and the rest worker processes
$ (used by PETSc)
$ You MUST launch MPI so that n MPI processes are created for each hardware node.
$
$ petscmpiexec -n 2 ./ex1 -hmpi_spawn_size 3 gives 2 application nodes (and 4 PETSc worker nodes)
$ petscmpiexec -n 6 ./ex1 -hmpi_merge_size 3 gives the SAME 2 application nodes and 4 PETSc worker nodes
$ This is what would use if each of the computers hardware nodes had 3 CPUs.
$
$ These are intended to be used in conjunction with USER HMPI code. The user will have 1 process per
$ computer (hardware) node (where the computer node has p cpus), the user's code will use threads to fully
$ utilize all the CPUs on the node. The PETSc code will have p processes to fully use the compute node for
$ PETSc calculations. The user THREADS and PETSc PROCESSES will NEVER run at the same time so the p CPUs
$ are always working on p task, never more than p.
$
$ See PCHMPI for a PETSc preconditioner that can use this functionality
$
For both PetscHMPISpawn() and PetscHMPIMerge() PETSC_COMM_WORLD consists of one process per "node", PETSC_COMM_LOCAL_WORLD
consists of all the processes in a "node."
In both cases the user's code is running ONLY on PETSC_COMM_WORLD (that was newly generated by running this command).
Level: developer
Concepts: HMPI
.seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscHMPIFinalize(), PetscInitialize(), PetscHMPIMerge(), PetscHMPIRun()
@*/
PetscErrorCode PetscHMPISpawn(PetscMPIInt nodesize)
{
PetscErrorCode ierr;
PetscMPIInt size;
MPI_Comm parent,children;
PetscFunctionBegin;
ierr = MPI_Comm_get_parent(&parent);CHKERRQ(ierr);
if (parent == MPI_COMM_NULL) { /* the original processes started by user */
char programname[PETSC_MAX_PATH_LEN];
char **argv;
ierr = PetscGetProgramName(programname,PETSC_MAX_PATH_LEN);CHKERRQ(ierr);
ierr = PetscGetArguments(&argv);CHKERRQ(ierr);
ierr = MPI_Comm_spawn(programname,argv,nodesize-1,MPI_INFO_NULL,0,PETSC_COMM_SELF,&children,MPI_ERRCODES_IGNORE);CHKERRQ(ierr);
ierr = PetscFreeArguments(argv);CHKERRQ(ierr);
ierr = MPI_Intercomm_merge(children,0,&PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscInfo2(0,"PETSc HMPI successfully spawned: number of nodes = %d node size = %d\n",size,nodesize);CHKERRQ(ierr);
saved_PETSC_COMM_WORLD = PETSC_COMM_WORLD;
} else { /* worker nodes that get spawned */
ierr = MPI_Intercomm_merge(parent,1,&PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr);
ierr = PetscHMPIHandle(PETSC_COMM_LOCAL_WORLD);CHKERRQ(ierr);
PetscHMPIWorker = PETSC_TRUE; /* so that PetscHMPIFinalize() will not attempt a broadcast from this process */
PetscEnd(); /* cannot continue into user code */
}
PetscFunctionReturn(0);
}
PetscErrorCode
Metos3DFlag(PetscBool flag, char *message)
{
PetscFunctionBegin;
if (flag == PETSC_FALSE) {
PetscPrintf(PETSC_COMM_WORLD, "\n");
PetscPrintf(PETSC_COMM_WORLD, "### ERROR:\n");
PetscPrintf(PETSC_COMM_WORLD, "### ERROR: %s\n", message);
PetscPrintf(PETSC_COMM_WORLD, "### ERROR:\n");
PetscPrintf(PETSC_COMM_WORLD, "\n");
PetscEnd();
}
PetscFunctionReturn(0);
}
int ReportParams(AppCtx *user)
/*---------------------------------------------------------------------*/
{
Parameter *param;
GridInfo *grid = user->grid;
int ierr, ierr_out=0;
ierr = PetscBagGetData(user->bag,(void**)¶m);
CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"---------------MOC test 1----------------\n");
PetscPrintf(PETSC_COMM_WORLD,"Prescribed wind, method of\n");
PetscPrintf(PETSC_COMM_WORLD,"characteristics advection, explicit time-\n");
PetscPrintf(PETSC_COMM_WORLD,"stepping.\n\n");
if (param->flow_type == 0) {
PetscPrintf(PETSC_COMM_WORLD,"Flow_type: %d (shear cell).\n\n",param->flow_type);
}
if (param->flow_type == 1) {
PetscPrintf(PETSC_COMM_WORLD,"Flow_type: %d (rigid body rotation).\n\n",param->flow_type);
}
if (param->verify) {
PetscPrintf(PETSC_COMM_WORLD," ** VERIFICATION RUN ** \n\n");
}
ierr = PetscPrintf(PETSC_COMM_WORLD," [ni,nj] = %d, %d [dx,dz] = %5.4g, %5.4g\n",grid->ni,grid->nj,grid->dx,grid->dz);
CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," t_max = %g, cfl = %g, dt = %5.4g,",param->t_max,param->cfl,param->dt);
CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," t_output = %g\n",param->t_output_interval);
CHKERRQ(ierr);
if (param->output_to_file) {
PetscPrintf(PETSC_COMM_WORLD,"Output File: Binary file \"%s\"\n",param->output_filename);
}
if (!param->output_to_file)
PetscPrintf(PETSC_COMM_WORLD,"Output File: NO OUTPUT!\n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"----------------------------------------\n");
CHKERRQ(ierr);
if (param->param_test) PetscEnd();
return ierr_out;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
SNES snes; /* nonlinear solver */
Vec Hu,r; /* solution, residual vectors */
Mat J; /* Jacobian matrix */
AppCtx user; /* user-defined work context */
PetscInt its, i, tmpxs, tmpxm; /* iteration count, index, etc. */
PetscReal tmp1, tmp2, tmp3, tmp4, tmp5,
errnorms[2], descaleNode[2];
PetscTruth eps_set = PETSC_FALSE, dump = PETSC_FALSE, exactinitial = PETSC_FALSE,
snes_mf_set, snes_fd_set;
MatFDColoring matfdcoloring = 0;
ISColoring iscoloring;
SNESConvergedReason reason; /* Check convergence */
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &user.rank); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"BODVARDSSON solves for thickness and velocity in 1D, steady ice stream\n"
" [run with -help for info and options]\n");CHKERRQ(ierr);
user.n = 3.0; /* Glen flow law exponent */
user.secpera = 31556926.0;
user.rho = 910.0; /* kg m^-3 */
user.rhow = 1028.0; /* kg m^-3 */
user.g = 9.81; /* m s^-2 */
/* ask Test N for its parameters, but only those we need to solve */
ierr = params_exactN(&(user.H0), &tmp1, &(user.xc), &tmp2, &tmp3, &tmp4, &tmp5,
&(user.Txc)); CHKERRQ(ierr);
/* regularize using strain rate of 1/xc per year */
user.epsilon = (1.0 / user.secpera) / user.xc;
/* tools for non-dimensionalizing to improve equation scaling */
user.scaleNode[0] = 1000.0; user.scaleNode[1] = 100.0 / user.secpera;
ierr = PetscOptionsTruth("-snes_mf","","",PETSC_FALSE,&snes_mf_set,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-snes_fd","","",PETSC_FALSE,&snes_fd_set,NULL);CHKERRQ(ierr);
if (!snes_mf_set && !snes_fd_set) {
PetscPrintf(PETSC_COMM_WORLD,
"\n***ERROR: bodvardsson needs one or zero of '-snes_mf', '-snes_fd'***\n\n"
"USAGE FOLLOWS ...\n\n%s",help);
PetscEnd();
}
if (snes_fd_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" using approximate Jacobian; finite-differencing using coloring\n");
CHKERRQ(ierr);
} else if (snes_mf_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" matrix free; no preconditioner\n"); CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" true Jacobian\n"); CHKERRQ(ierr);
}
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,
"bodvardsson program options",__FILE__);CHKERRQ(ierr);
{
ierr = PetscOptionsTruth("-bod_up_one","","",PETSC_FALSE,&user.upwind1,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-bod_exact_init","","",PETSC_FALSE,&exactinitial,NULL);CHKERRQ(ierr);
ierr = PetscOptionsTruth("-bod_dump",
"dump out exact and approximate solution and residual, as asci","",
dump,&dump,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-bod_epsilon","regularization (a strain rate in units of 1/a)","",
user.epsilon * user.secpera,&user.epsilon,&eps_set);CHKERRQ(ierr);
if (eps_set) user.epsilon *= 1.0 / user.secpera;
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
/* Create machinery for parallel grid management (DA), nonlinear solver (SNES),
and Vecs for fields (solution, RHS). Note default Mx=46 grid points means
dx=10 km. Also degrees of freedom = 2 (thickness and velocity
at each point) and stencil radius = ghost width = 2 for 2nd-order upwinding. */
user.solnghostwidth = 2;
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,-46,2,user.solnghostwidth,PETSC_NULL,&user.da);
CHKERRQ(ierr);
ierr = DASetUniformCoordinates(user.da,0.0,user.xc,
PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = DASetFieldName(user.da,0,"ice thickness [non-dimensional]"); CHKERRQ(ierr);
ierr = DASetFieldName(user.da,1,"ice velocity [non-dimensional]"); CHKERRQ(ierr);
ierr = DAGetInfo(user.da,PETSC_IGNORE,&user.Mx,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);
ierr = DAGetCorners(user.da,&user.xs,PETSC_NULL,PETSC_NULL,&user.xm,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
user.dx = user.xc / (PetscReal)(user.Mx-1);
/* another DA for scalar parameters, with same length */
ierr = DACreate1d(PETSC_COMM_WORLD,DA_NONPERIODIC,user.Mx,1,1,PETSC_NULL,&user.scalarda);CHKERRQ(ierr);
ierr = DASetUniformCoordinates(user.scalarda,0.0,user.xc,
PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
/* check that parallel layout of scalar DA is same as dof=2 DA */
ierr = DAGetCorners(user.scalarda,&tmpxs,PETSC_NULL,PETSC_NULL,&tmpxm,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
if ((tmpxs != user.xs) || (tmpxm != user.xm)) {
PetscPrintf(PETSC_COMM_SELF,
"\n***ERROR: rank %d gets different ownership range for the two DAs! ENDING ...***\n\n",
user.rank);
PetscEnd();
}
ierr = PetscPrintf(PETSC_COMM_WORLD,
" Mx = %D points, dx = %.3f m\n H0 = %.2f m, xc = %.2f km, Txc = %.5e Pa m\n",
user.Mx, user.dx, user.H0, user.xc/1000.0, user.Txc);CHKERRQ(ierr);
/* Extract/allocate global vectors from DAs and duplicate for remaining same types */
ierr = DACreateGlobalVector(user.da,&Hu);CHKERRQ(ierr);
ierr = VecSetBlockSize(Hu,2);CHKERRQ(ierr);
ierr = VecDuplicate(Hu,&r);CHKERRQ(ierr); /* inherits block size */
ierr = VecDuplicate(Hu,&user.Huexact);CHKERRQ(ierr); /* ditto */
ierr = DACreateGlobalVector(user.scalarda,&user.M);CHKERRQ(ierr);
ierr = VecDuplicate(user.M,&user.Bstag);CHKERRQ(ierr);
ierr = VecDuplicate(user.M,&user.beta);CHKERRQ(ierr);
ierr = DASetLocalFunction(user.da,(DALocalFunction1)scshell);CHKERRQ(ierr);
ierr = DASetLocalJacobian(user.da,(DALocalFunction1)BodJacobianMatrixLocal);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
ierr = SNESSetFunction(snes,r,SNESDAFormFunction,&user);CHKERRQ(ierr);
/* setting up a matrix is only actually needed for -snes_fd case */
ierr = DAGetMatrix(user.da,MATAIJ,&J);CHKERRQ(ierr);
if (snes_fd_set) {
/* tools needed so DA can use sparse matrix for its F.D. Jacobian approx */
ierr = DAGetColoring(user.da,IS_COLORING_GLOBAL,MATAIJ,&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringCreate(J,iscoloring,&matfdcoloring);CHKERRQ(ierr);
ierr = ISColoringDestroy(iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringSetFunction(matfdcoloring,
(PetscErrorCode (*)(void))SNESDAFormFunction,&user);CHKERRQ(ierr);
ierr = MatFDColoringSetFromOptions(matfdcoloring);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes,J,J,SNESDefaultComputeJacobianColor,matfdcoloring);CHKERRQ(ierr);
} else {
ierr = SNESSetJacobian(snes,J,J,SNESDAComputeJacobian,&user);CHKERRQ(ierr);
}
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
/* the the Bodvardsson (1955) exact solution allows setting M(x), B(x), beta(x), T(xc) */
ierr = FillDistributedParams(&user);CHKERRQ(ierr);
/* the exact thickness and exact ice velocity (user.uHexact) are known from Bodvardsson (1955) */
ierr = FillExactSoln(&user); CHKERRQ(ierr);
if (exactinitial) {
ierr = PetscPrintf(PETSC_COMM_WORLD," using exact solution as initial guess\n");
CHKERRQ(ierr);
/* the initial guess is the exact continuum solution */
ierr = VecCopy(user.Huexact,Hu); CHKERRQ(ierr);
} else {
ierr = FillInitial(&user, &Hu); CHKERRQ(ierr);
}
/************ SOLVE NONLINEAR SYSTEM ************/
/* recall that RHS r is used internally by KSP, and is set by the SNES */
for (i = 0; i < 2; i++) descaleNode[i] = 1.0 / user.scaleNode[i];
ierr = VecStrideScaleAll(Hu,descaleNode); CHKERRQ(ierr); /* de-dimensionalize initial guess */
ierr = SNESSolve(snes,PETSC_NULL,Hu);CHKERRQ(ierr);
ierr = VecStrideScaleAll(Hu,user.scaleNode); CHKERRQ(ierr); /* put back in "real" scale */
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = SNESGetConvergedReason(snes,&reason);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" %s Number of Newton iterations = %D\n",
SNESConvergedReasons[reason],its);CHKERRQ(ierr);
if (dump) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing combined result Hu\n");CHKERRQ(ierr);
ierr = VecView(Hu,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing combined exact result Huexact\n");CHKERRQ(ierr);
ierr = VecView(user.Huexact,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" viewing final combined residual at Hu\n");CHKERRQ(ierr);
ierr = VecView(r,PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
}
/* evaluate error relative to exact solution */
ierr = VecAXPY(Hu,-1.0,user.Huexact); CHKERRQ(ierr); /* Hu = - Huexact + Hu */
ierr = VecStrideNormAll(Hu,NORM_INFINITY,errnorms); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"(dx,errHinf,erruinf) %.3f %.4e %.4e\n",
user.dx,errnorms[0],errnorms[1]*user.secpera);CHKERRQ(ierr);
ierr = VecDestroy(Hu);CHKERRQ(ierr);
ierr = VecDestroy(r);CHKERRQ(ierr);
ierr = VecDestroy(user.Huexact);CHKERRQ(ierr);
ierr = VecDestroy(user.M);CHKERRQ(ierr);
ierr = VecDestroy(user.Bstag);CHKERRQ(ierr);
ierr = VecDestroy(user.beta);CHKERRQ(ierr);
ierr = MatDestroy(J); CHKERRQ(ierr);
ierr = SNESDestroy(snes);CHKERRQ(ierr);
ierr = DADestroy(user.da);CHKERRQ(ierr);
ierr = DADestroy(user.scalarda);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
/*@C
PetscInitialize - Initializes the PETSc database and MPI.
PetscInitialize() calls MPI_Init() if that has yet to be called,
so this routine should always be called near the beginning of
your program -- usually the very first line!
Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set
Input Parameters:
+ argc - count of number of command line arguments
. args - the command line arguments
. file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use NULL to not check for
code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files
- help - [optional] Help message to print, use NULL for no message
If you wish PETSc code to run ONLY on a subcommunicator of MPI_COMM_WORLD, create that
communicator first and assign it to PETSC_COMM_WORLD BEFORE calling PetscInitialize(). Thus if you are running a
four process job and two processes will run PETSc and have PetscInitialize() and PetscFinalize() and two process will not,
then do this. If ALL processes in the job are using PetscInitialize() and PetscFinalize() then you don't need to do this, even
if different subcommunicators of the job are doing different things with PETSc.
Options Database Keys:
+ -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger
. -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected
. -on_error_emacs <machinename> causes emacsclient to jump to error file
. -on_error_abort calls abort() when error detected (no traceback)
. -on_error_mpiabort calls MPI_abort() when error detected
. -error_output_stderr prints error messages to stderr instead of the default stdout
. -error_output_none does not print the error messages (but handles errors in the same way as if this was not called)
. -debugger_nodes [node1,node2,...] - Indicates nodes to start in debugger
. -debugger_pause [sleeptime] (in seconds) - Pauses debugger
. -stop_for_debugger - Print message on how to attach debugger manually to
process and wait (-debugger_pause) seconds for attachment
. -malloc - Indicates use of PETSc error-checking malloc (on by default for debug version of libraries)
. -malloc no - Indicates not to use error-checking malloc
. -malloc_debug - check for memory corruption at EVERY malloc or free
. -malloc_test - like -malloc_dump -malloc_debug, but only active for debugging builds
. -fp_trap - Stops on floating point exceptions (Note that on the
IBM RS6000 this slows code by at least a factor of 10.)
. -no_signal_handler - Indicates not to trap error signals
. -shared_tmp - indicates /tmp directory is shared by all processors
. -not_shared_tmp - each processor has own /tmp
. -tmp - alternative name of /tmp directory
. -get_total_flops - returns total flops done by all processors
. -memory_info - Print memory usage at end of run
- -server <port> - start PETSc webserver (default port is 8080)
Options Database Keys for Profiling:
See the <a href="../../docs/manual.pdf#nameddest=ch_profiling">profiling chapter of the users manual</a> for details.
+ -info <optional filename> - Prints verbose information to the screen
. -info_exclude <null,vec,mat,pc,ksp,snes,ts> - Excludes some of the verbose messages
. -log_sync - Log the synchronization in scatters, inner products and norms
. -log_trace [filename] - Print traces of all PETSc calls to the screen (useful to determine where a program
hangs without running in the debugger). See PetscLogTraceBegin().
. -log_summary [filename] - Prints summary of flop and timing information to screen. If the filename is specified the
summary is written to the file. See PetscLogView().
. -log_summary_python [filename] - Prints data on of flop and timing usage to a file or screen. See PetscLogPrintSViewPython().
. -log_all [filename] - Logs extensive profiling information See PetscLogDump().
. -log [filename] - Logs basic profiline information See PetscLogDump().
- -log_mpe [filename] - Creates a logfile viewable by the utility Jumpshot (in MPICH distribution)
Only one of -log_trace, -log_summary, -log_all, -log, or -log_mpe may be used at a time
Environmental Variables:
+ PETSC_TMP - alternative tmp directory
. PETSC_SHARED_TMP - tmp is shared by all processes
. PETSC_NOT_SHARED_TMP - each process has its own private tmp
. PETSC_VIEWER_SOCKET_PORT - socket number to use for socket viewer
- PETSC_VIEWER_SOCKET_MACHINE - machine to use for socket viewer to connect to
Level: beginner
Notes:
If for some reason you must call MPI_Init() separately, call
it before PetscInitialize().
Fortran Version:
In Fortran this routine has the format
$ call PetscInitialize(file,ierr)
+ ierr - error return code
- file - [optional] PETSc database file, also checks ~username/.petscrc and .petscrc use NULL_CHARACTER to not check for
code specific file. Use -skip_petscrc in the code specific file to skip the .petscrc files
Important Fortran Note:
In Fortran, you MUST use NULL_CHARACTER to indicate a
null character string; you CANNOT just use NULL as
in the C version. See the <a href="../../docs/manual.pdf">users manual</a> for details.
If your main program is C but you call Fortran code that also uses PETSc you need to call PetscInitializeFortran() soon after
calling PetscInitialize().
Concepts: initializing PETSc
.seealso: PetscFinalize(), PetscInitializeFortran(), PetscGetArgs(), PetscInitializeNoArguments()
@*/
PetscErrorCode PetscInitialize(int *argc,char ***args,const char file[],const char help[])
{
PetscErrorCode ierr;
PetscMPIInt flag, size;
PetscInt nodesize;
PetscBool flg;
char hostname[256];
PetscFunctionBegin;
if (PetscInitializeCalled) PetscFunctionReturn(0);
/* these must be initialized in a routine, not as a constant declaration*/
PETSC_STDOUT = stdout;
PETSC_STDERR = stderr;
ierr = PetscOptionsCreate();CHKERRQ(ierr);
/*
We initialize the program name here (before MPI_Init()) because MPICH has a bug in
it that it sets args[0] on all processors to be args[0] on the first processor.
*/
if (argc && *argc) {
ierr = PetscSetProgramName(**args);CHKERRQ(ierr);
} else {
ierr = PetscSetProgramName("Unknown Name");CHKERRQ(ierr);
}
ierr = MPI_Initialized(&flag);CHKERRQ(ierr);
if (!flag) {
if (PETSC_COMM_WORLD != MPI_COMM_NULL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"You cannot set PETSC_COMM_WORLD if you have not initialized MPI first");
#if defined(PETSC_HAVE_MPI_INIT_THREAD)
{
PetscMPIInt provided;
ierr = MPI_Init_thread(argc,args,MPI_THREAD_FUNNELED,&provided);CHKERRQ(ierr);
}
#else
ierr = MPI_Init(argc,args);CHKERRQ(ierr);
#endif
PetscBeganMPI = PETSC_TRUE;
}
if (argc && args) {
PetscGlobalArgc = *argc;
PetscGlobalArgs = *args;
}
PetscFinalizeCalled = PETSC_FALSE;
if (PETSC_COMM_WORLD == MPI_COMM_NULL) PETSC_COMM_WORLD = MPI_COMM_WORLD;
ierr = MPI_Comm_set_errhandler(PETSC_COMM_WORLD,MPI_ERRORS_RETURN);CHKERRQ(ierr);
/* Done after init due to a bug in MPICH-GM? */
ierr = PetscErrorPrintfInitialize();CHKERRQ(ierr);
ierr = MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank);CHKERRQ(ierr);
ierr = MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize);CHKERRQ(ierr);
MPIU_BOOL = MPI_INT;
MPIU_ENUM = MPI_INT;
/*
Initialized the global complex variable; this is because with
shared libraries the constructors for global variables
are not called; at least on IRIX.
*/
#if defined(PETSC_HAVE_COMPLEX)
{
#if defined(PETSC_CLANGUAGE_CXX)
PetscComplex ic(0.0,1.0);
PETSC_i = ic;
#elif defined(PETSC_CLANGUAGE_C)
PETSC_i = _Complex_I;
#endif
}
#if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr);
ierr = MPI_Type_commit(&MPIU_C_DOUBLE_COMPLEX);CHKERRQ(ierr);
ierr = MPI_Type_contiguous(2,MPI_FLOAT,&MPIU_C_COMPLEX);CHKERRQ(ierr);
ierr = MPI_Type_commit(&MPIU_C_COMPLEX);CHKERRQ(ierr);
#endif
#endif /* PETSC_HAVE_COMPLEX */
/*
Create the PETSc MPI reduction operator that sums of the first
half of the entries and maxes the second half.
*/
ierr = MPI_Op_create(PetscMaxSum_Local,1,&PetscMaxSum_Op);CHKERRQ(ierr);
#if defined(PETSC_USE_REAL___FLOAT128)
ierr = MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU___FLOAT128);CHKERRQ(ierr);
ierr = MPI_Type_commit(&MPIU___FLOAT128);CHKERRQ(ierr);
#if defined(PETSC_HAVE_COMPLEX)
ierr = MPI_Type_contiguous(4,MPI_DOUBLE,&MPIU___COMPLEX128);CHKERRQ(ierr);
ierr = MPI_Type_commit(&MPIU___COMPLEX128);CHKERRQ(ierr);
#endif
ierr = MPI_Op_create(PetscMax_Local,1,&MPIU_MAX);CHKERRQ(ierr);
ierr = MPI_Op_create(PetscMin_Local,1,&MPIU_MIN);CHKERRQ(ierr);
#endif
#if (defined(PETSC_HAVE_COMPLEX) && !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)) || defined(PETSC_USE_REAL___FLOAT128)
ierr = MPI_Op_create(PetscSum_Local,1,&MPIU_SUM);CHKERRQ(ierr);
#endif
ierr = MPI_Type_contiguous(2,MPIU_SCALAR,&MPIU_2SCALAR);CHKERRQ(ierr);
ierr = MPI_Type_commit(&MPIU_2SCALAR);CHKERRQ(ierr);
ierr = MPI_Op_create(PetscADMax_Local,1,&PetscADMax_Op);CHKERRQ(ierr);
ierr = MPI_Op_create(PetscADMin_Local,1,&PetscADMin_Op);CHKERRQ(ierr);
#if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
ierr = MPI_Type_contiguous(2,MPIU_INT,&MPIU_2INT);CHKERRQ(ierr);
ierr = MPI_Type_commit(&MPIU_2INT);CHKERRQ(ierr);
#endif
/*
Attributes to be set on PETSc communicators
*/
ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelCounter,&Petsc_Counter_keyval,(void*)0);CHKERRQ(ierr);
ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm_Outer,&Petsc_InnerComm_keyval,(void*)0);CHKERRQ(ierr);
ierr = MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelComm_Inner,&Petsc_OuterComm_keyval,(void*)0);CHKERRQ(ierr);
/*
Build the options database
*/
ierr = PetscOptionsInsert(argc,args,file);CHKERRQ(ierr);
/*
Print main application help message
*/
ierr = PetscOptionsHasName(NULL,"-help",&flg);CHKERRQ(ierr);
if (help && flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD,help);CHKERRQ(ierr);
}
ierr = PetscOptionsCheckInitial_Private();CHKERRQ(ierr);
/* SHOULD PUT IN GUARDS: Make sure logging is initialized, even if we do not print it out */
#if defined(PETSC_USE_LOG)
ierr = PetscLogBegin_Private();CHKERRQ(ierr);
#endif
/*
Load the dynamic libraries (on machines that support them), this registers all
the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes)
*/
ierr = PetscInitialize_DynamicLibraries();CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscInfo1(0,"PETSc successfully started: number of processors = %d\n",size);CHKERRQ(ierr);
ierr = PetscGetHostName(hostname,256);CHKERRQ(ierr);
ierr = PetscInfo1(0,"Running on machine: %s\n",hostname);CHKERRQ(ierr);
ierr = PetscOptionsCheckInitial_Components();CHKERRQ(ierr);
/* Check the options database for options related to the options database itself */
ierr = PetscOptionsSetFromOptions();CHKERRQ(ierr);
#if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
/*
Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI
Currently not used because it is not supported by MPICH.
*/
#if !defined(PETSC_WORDS_BIGENDIAN)
ierr = MPI_Register_datarep((char*)"petsc",PetscDataRep_read_conv_fn,PetscDataRep_write_conv_fn,PetscDataRep_extent_fn,NULL);CHKERRQ(ierr);
#endif
#endif
ierr = PetscOptionsGetInt(NULL,"-hmpi_spawn_size",&nodesize,&flg);CHKERRQ(ierr);
if (flg) {
#if defined(PETSC_HAVE_MPI_COMM_SPAWN)
ierr = PetscHMPISpawn((PetscMPIInt) nodesize);CHKERRQ(ierr); /* worker nodes never return from here; they go directly to PetscEnd() */
#else
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"PETSc built without MPI 2 (MPI_Comm_spawn) support, use -hmpi_merge_size instead");
#endif
} else {
ierr = PetscOptionsGetInt(NULL,"-hmpi_merge_size",&nodesize,&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscHMPIMerge((PetscMPIInt) nodesize,NULL,NULL);CHKERRQ(ierr);
if (PetscHMPIWorker) { /* if worker then never enter user code */
PetscInitializeCalled = PETSC_TRUE;
PetscEnd();
}
}
}
#if defined(PETSC_HAVE_CUDA)
{
PetscMPIInt p;
for (p = 0; p < PetscGlobalSize; ++p) {
if (p == PetscGlobalRank) cublasInit();
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
}
#endif
ierr = PetscOptionsHasName(NULL,"-python",&flg);CHKERRQ(ierr);
if (flg) {
PetscInitializeCalled = PETSC_TRUE;
ierr = PetscPythonInitialize(NULL,NULL);CHKERRQ(ierr);
}
ierr = PetscThreadCommInitializePackage();CHKERRQ(ierr);
/*
Setup building of stack frames for all function calls
*/
#if defined(PETSC_USE_DEBUG)
PetscThreadLocalRegister((PetscThreadKey*)&petscstack); /* Creates petscstack_key if needed */
ierr = PetscStackCreate();CHKERRQ(ierr);
#endif
#if defined(PETSC_SERIALIZE_FUNCTIONS)
ierr = PetscFPTCreate(10000);CHKERRQ(ierr);
#endif
/*
Once we are completedly initialized then we can set this variables
*/
PetscInitializeCalled = PETSC_TRUE;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
SNES snes; /* nonlinear solver */
Vec Hu; /* solution vector */
AppCtx user; /* user-defined work context */
ExactCtx exact;
PetscInt its; /* snes reports iteration count */
SNESConvergedReason reason; /* snes reports convergence */
PetscReal tmp1, tmp2, tmp3,
errnorms[2], scaleNode[2], descaleNode[2];
PetscInt i;
char dumpfile[80],dxdocstr[80];
PetscBool eps_set = PETSC_FALSE,
dump = PETSC_FALSE,
exactinitial = PETSC_FALSE,
snes_mf_set, snes_fd_set, dx_set;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &user.rank); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"MARINE solves for thickness and velocity in 1D, steady marine ice sheet\n"
" [run with -help for info and options]\n");CHKERRQ(ierr);
user.n = 3.0; /* Glen flow law exponent */
user.secpera = 31556926.0;
user.rho = 910.0; /* kg m^-3 */
user.rhow = 1028.0; /* kg m^-3 */
user.omega = 1.0 - user.rho / user.rhow;
user.g = 9.81; /* m s^-2 */
/* get parameters of exact solution */
ierr = params_exactBod(&tmp1, &(exact.L0), &(exact.xg), &tmp2, &tmp3, &(user.k)); CHKERRQ(ierr);
ierr = exactBod(exact.xg,&(exact.Hg),&tmp2,&(exact.Mg)); CHKERRQ(ierr);
ierr = exactBodBueler(exact.xg,&tmp1,&(exact.Bg)); CHKERRQ(ierr);
user.zocean = user.rho * exact.Hg / user.rhow;
/* see ../marineshoot.py: */
#define xa_default 0.2
#define xc_default 0.98
/* define interval [xa,xc] */
user.xa = xa_default * exact.L0;
user.xc = xc_default * exact.L0;
/* get Dirichlet boundary conditions, and mass balance on shelf */
ierr = exactBod(user.xa, &(user.Ha), &(user.ua), &tmp1); CHKERRQ(ierr);
/* regularize using strain rate of 1/(length) per year */
user.epsilon = (1.0 / user.secpera) / (user.xc - user.xa);
/*user.epsilon = 0.0;*/
user.Hscale = 1000.0;
user.uscale = 100.0 / user.secpera;
user.noscale = PETSC_FALSE;
user.dx = 10000.0; /* default to coarse 10 km grid */
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,
"","options to marine (steady marine ice sheet solver)","");CHKERRQ(ierr);
{
ierr = PetscOptionsBool("-snes_mf","","",PETSC_FALSE,&snes_mf_set,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-snes_fd","","",PETSC_FALSE,&snes_fd_set,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-noscale","","",PETSC_FALSE,&user.noscale,NULL);CHKERRQ(ierr);
snprintf(dxdocstr,80,"target grid spacing (m) on interval of length %.0f m",
user.xc-user.xa);
ierr = PetscOptionsReal("-dx",dxdocstr,"",user.dx,&user.dx,&dx_set);CHKERRQ(ierr);
ierr = PetscOptionsBool("-exactinit",
"initialize using exact solution instead of default linear function","",
PETSC_FALSE,&exactinitial,NULL);CHKERRQ(ierr);
ierr = PetscOptionsString("-dump",
"dump approx and exact solution into given file (as ascii matlab format)","",
NULL,dumpfile,80,&dump);CHKERRQ(ierr);
ierr = PetscOptionsReal("-epsilon","regularizing strain rate for stress computation (a-1)","",
user.epsilon * user.secpera,&user.epsilon,&eps_set);CHKERRQ(ierr);
if (eps_set) user.epsilon *= 1.0 / user.secpera;
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
if (snes_fd_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" using approximate Jacobian; finite-differencing using coloring\n");
CHKERRQ(ierr);
} else if (snes_mf_set) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" matrix free; no preconditioner\n"); CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,
" true Jacobian\n"); CHKERRQ(ierr);
}
if (dx_set && (user.dx <= 0.0)) {
PetscPrintf(PETSC_COMM_WORLD,
"\n***ERROR: -dx value must be positive ... USAGE FOLLOWS ...\n\n%s",help);
PetscEnd();
}
user.N = (int)ceil( ((user.xc - user.xa) / user.dx) - 0.5 );
user.Mx = user.N + 2;
user.dx = (user.xc - user.xa) / ((PetscReal)(user.N) + 0.5); /* recompute so dx * (N+1/2) = xc - xa */
if (dx_set && (user.dx < 1.0)) {
PetscPrintf(PETSC_COMM_WORLD,
"\n***WARNING: '-dx %.3f' meters is below one meter and creates grid of %d points\n"
" ... probably uncomputable!\n\n",
user.dx,user.Mx);
}
/* residual scaling coeffs; motivation at right */
user.rscHa = 1.0 / user.Hscale, /* Dirichlet cond for H */
user.rscua = 1.0 / user.uscale, /* Dirichlet cond for u */
user.rscuH = user.dx / (user.Hscale * user.uscale), /* flux derivative d(uH)/dx */
user.rscstress = 1.0 / (user.k * user.rho * user.g * user.Hscale * user.uscale), /* beta term in SSA */
user.rsccalv = 1.0 / (0.025 * user.rho * user.g * user.Hscale); /* 0.5 * omega * overburden */
/* Create machinery for parallel grid management (DMDA), nonlinear solver (SNES),
and Vecs for fields (solution, RHS). Degrees of freedom = 2 (thickness and
velocity at each point). */
ierr = DMDACreate1d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,user.Mx,2,1,PETSC_NULL,&user.da);
CHKERRQ(ierr);
ierr = DMSetApplicationContext(user.da,&user);CHKERRQ(ierr);
ierr = DMDASetFieldName(user.da,0,"ice thickness [non-dimensional]"); CHKERRQ(ierr);
ierr = DMDASetFieldName(user.da,1,"ice velocity [non-dimensional]"); CHKERRQ(ierr);
ierr = DMSetFromOptions(user.da); CHKERRQ(ierr);
ierr = DMDAGetInfo(user.da,PETSC_IGNORE,&user.Mx,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE,
PETSC_IGNORE,PETSC_IGNORE,PETSC_IGNORE);CHKERRQ(ierr);
ierr = DMDAGetCorners(user.da,&user.xs,PETSC_NULL,PETSC_NULL,&user.xm,PETSC_NULL,PETSC_NULL);
CHKERRQ(ierr);
/* another DMDA for scalar staggered parameters; one fewer point */
ierr = DMDACreate1d(PETSC_COMM_WORLD,DMDA_BOUNDARY_NONE,user.Mx-1,1,1,PETSC_NULL,&user.stagda);
CHKERRQ(ierr);
/* establish geometry on grid; note xa = x_0 and xc = x_{N+1/2} */
ierr = DMDASetUniformCoordinates(user.da, user.xa, user.xc+0.5*user.dx,
0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(user.stagda,user.xa+0.5*user.dx,user.xc,
0.0,1.0,0.0,1.0);CHKERRQ(ierr);
/* report on current grid */
ierr = PetscPrintf(PETSC_COMM_WORLD,
" grid: Mx = N+2 = %D regular points, dx = %.3f m, xa = %.2f km, xc = %.2f km\n",
user.Mx, user.dx, user.xa/1000.0, user.xc/1000.0);CHKERRQ(ierr);
/* Extract/allocate global vectors from DMDAs and duplicate for remaining same types */
ierr = DMCreateGlobalVector(user.da,&Hu);CHKERRQ(ierr);
ierr = VecSetBlockSize(Hu,2);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)Hu,"Hu");CHKERRQ(ierr);
ierr = VecDuplicate(Hu,&exact.Hu);CHKERRQ(ierr); /* inherits block size */
ierr = PetscObjectSetName((PetscObject)exact.Hu,"exactHu");CHKERRQ(ierr);
ierr = DMCreateGlobalVector(user.stagda,&user.Mstag);CHKERRQ(ierr);
ierr = VecDuplicate(user.Mstag,&user.Bstag);CHKERRQ(ierr);
/* set up snes */
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
ierr = SNESSetDM(snes,user.da);CHKERRQ(ierr);
ierr = DMDASNESSetFunctionLocal(user.da,INSERT_VALUES,(DMDASNESFunction)scshell,&user);CHKERRQ(ierr);
ierr = DMDASNESSetJacobianLocal(user.da,(DMDASNESJacobian)JacobianMatrixLocal,&user);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
/* the exact thickness and exact ice velocity (user.uHexact) are known */
ierr = FillExactSoln(&exact, &user); CHKERRQ(ierr);
/* the exact solution allows setting M(x), B(x) */
ierr = FillDistributedParams(&exact, &user);CHKERRQ(ierr);
if (exactinitial) {
ierr = PetscPrintf(PETSC_COMM_WORLD," using exact solution as initial guess\n"); CHKERRQ(ierr);
/* the initial guess is the exact continuum solution */
ierr = VecCopy(exact.Hu,Hu); CHKERRQ(ierr);
} else {
/* the initial guess is a linear solution */
ierr = FillInitial(&user, &Hu); CHKERRQ(ierr);
}
/************ SOLVE NONLINEAR SYSTEM ************/
/* recall that RHS r is used internally by KSP, and is set by the SNES */
if (user.noscale) {
user.Hscale = 1.0;
user.uscale = 1.0;
}
scaleNode[0] = user.Hscale;
scaleNode[1] = user.uscale;
for (i = 0; i < 2; i++) descaleNode[i] = 1.0 / scaleNode[i];
ierr = VecStrideScaleAll(Hu,descaleNode); CHKERRQ(ierr); /* de-dimensionalize initial guess */
ierr = SNESSolve(snes,PETSC_NULL,Hu);CHKERRQ(ierr);
ierr = VecStrideScaleAll(Hu,scaleNode); CHKERRQ(ierr); /* put back in "real" scale */
/* minimal report on solve */
ierr = SNESGetIterationNumber(snes,&its);CHKERRQ(ierr);
ierr = SNESGetConvergedReason(snes,&reason);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
" %s Number of Newton iterations = %D\n",
SNESConvergedReasons[reason],its);CHKERRQ(ierr);
if (dump) {
ierr = PetscPrintf(PETSC_COMM_WORLD,
"dumping results in ascii matlab format to file '%s' ...\n",dumpfile);CHKERRQ(ierr);
PetscViewer viewer;
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,dumpfile,&viewer);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);CHKERRQ(ierr);
DM coord_da;
Vec coord_x;
ierr = DMGetCoordinateDM(user.da, &coord_da); CHKERRQ(ierr);
ierr = DMGetCoordinates(user.da, &coord_x); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% START MATLAB\n"); CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)coord_x,"x");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% viewing coordinate vector x \n");CHKERRQ(ierr);
ierr = VecView(coord_x,viewer); CHKERRQ(ierr);
ierr = VecView(Hu,viewer); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% viewing combined result Hu\n");CHKERRQ(ierr);
ierr = VecView(Hu,viewer); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% viewing combined exact result exactHu\n");CHKERRQ(ierr);
ierr = VecView(exact.Hu,viewer); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,
"%% defining plottable variables and plotting in Matlab\n"
"Mx = %d; secpera = 31556926.0;\n"
"H = Hu(1:2:2*Mx-1);\n"
"u = Hu(2:2:2*Mx);\n"
"exactH = exactHu(1:2:2*Mx-1);\n"
"exactu = exactHu(2:2:2*Mx);\n"
"figure, plot(x,H,x,exactH);\n"
"legend('numerical','exact'), xlabel x, ylabel('H (m)')\n"
"figure, plot(x,u*secpera,x,exactu*secpera);\n"
"legend('numerical','exact'), xlabel x, ylabel('u (m/a)')\n"
"figure, subplot(2,1,1), semilogy(x,abs(H-exactH));\n"
"ylabel('H error (m)'), grid\n"
"subplot(2,1,2), semilogy(x,abs(u-exactu)*secpera);\n"
"xlabel x, ylabel('u error (m/a)'), grid\n",
user.Mx); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%% END MATLAB\n"); CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
/* evaluate error relative to exact solution */
ierr = VecAXPY(Hu,-1.0,exact.Hu); CHKERRQ(ierr); /* Hu = - Huexact + Hu */
ierr = VecStrideNormAll(Hu,NORM_INFINITY,errnorms); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"(dx,errHinf,erruinf) %.3f %.4e %.4e\n",
user.dx,errnorms[0],errnorms[1]*user.secpera);CHKERRQ(ierr);
ierr = VecDestroy(&Hu);CHKERRQ(ierr);
ierr = VecDestroy(&(exact.Hu));CHKERRQ(ierr);
ierr = VecDestroy(&(user.Mstag));CHKERRQ(ierr);
ierr = VecDestroy(&(user.Bstag));CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = DMDestroy(&(user.da));CHKERRQ(ierr);
ierr = DMDestroy(&(user.stagda));CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
int main(int argc,char **args)
{
PetscErrorCode ierr;
PetscInitialize(&argc,&args,(char *)0,help);
PetscInt m = 10; /* default number of rows and columns IN GRID,
but matrix is (m^2) x (m^2) */
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
MPI_Comm com = PETSC_COMM_WORLD;
PetscMPIInt rank, size;
ierr = MPI_Comm_rank(com, &rank); CHKERRQ(ierr);
ierr = MPI_Comm_size(com, &size); CHKERRQ(ierr);
/* create m x m two-dimensional grid for periodic boundary condition problem */
DA da2;
PetscInt dof=1, stencilwidth=1;
ierr = DACreate2d(com, DA_XYPERIODIC, DA_STENCIL_STAR,
m,m,PETSC_DECIDE,PETSC_DECIDE,
dof,stencilwidth,PETSC_NULL,PETSC_NULL,&da2); CHKERRQ(ierr);
/* get da2-managed Vecs */
Vec x,b,u;
ierr = DACreateGlobalVector(da2,&x); CHKERRQ(ierr);
ierr = VecDuplicate(x,&b); CHKERRQ(ierr);
ierr = VecDuplicate(x,&u); CHKERRQ(ierr);
Mat A;
ierr = DAGetMatrix(da2, MATMPIAIJ, &A); CHKERRQ(ierr);
/* alternative call below is not quite same as result from DAGetMatrix(),
because of nonzero allocation; the Mat ownership ranges are same */
/* ierr = MatCreateMPIAIJ(com, mlocal, mlocal, m*m, m*m,
5, PETSC_NULL, 4, PETSC_NULL,
&A); CHKERRQ(ierr) */
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
/* report on ownership range */
PetscInt rstart,rend,mlocal;
ierr = VecGetOwnershipRange(x,&rstart,&rend);CHKERRQ(ierr);
ierr = VecGetLocalSize(x,&mlocal);CHKERRQ(ierr);
PetscInt A_rstart,A_rend;
ierr = MatGetOwnershipRange(A,&A_rstart,&A_rend);CHKERRQ(ierr);
if ((rstart != A_rstart) || (rend != A_rend)) {
ierr = PetscPrintf(com,
"Vec and Mat ownership ranges different!!! ending ...\n"); CHKERRQ(ierr);
PetscEnd();
} else {
ierr = PetscSynchronizedPrintf(com,
"rank=%d has Vec and Mat ownership: mlocal=%d, rstart=%d, rend=%d\n",
rank,mlocal,rstart,rend); CHKERRQ(ierr);
}
PetscSynchronizedFlush(com);
/* get local part of grid */
PetscInt xm,ym,xs,ys;
DAGetCorners(da2,&xs,&ys,0,&xm,&ym,0);
/* report on local part of grid */
ierr = PetscSynchronizedPrintf(com,
"rank=%d has da2-managed-Vec local ranges: xs=%d, xm=%d, ys=%d, ym=%d\n",
rank,xs,xm,ys,ym); CHKERRQ(ierr);
PetscSynchronizedFlush(com);
/* set up linear system */
PetscScalar **barr, **uarr; /* RHS and exact soln, resp. */
DAVecGetArray(da2, b, &barr);
DAVecGetArray(da2, u, &uarr);
PetscScalar dx = 1.0/(double)m, dy = dx, pi = 3.14159265358979;
PetscScalar xi,yj;
PetscInt diag=0,north=1,east=2,south=3,west=4;
PetscScalar vals[5] = {-4.0 + dx * dx, 1.0, 1.0, 1.0, 1.0};
MatStencil row, col[5]; /* these are not "stencils" at all, but local grid
to global indices helpers */
PetscInt i,j,num;
for (j=ys; j<ys+ym; j++) {
for(i=xs; i<xs+xm; i++) {
/* entries of matrix A */
row.i = i; row.j = j; row.c = 0; /* dof = 1 so first component;
note row.k is for 3d DAs */
for (num=0; num<5; num++) col[num].c = 0;
/* set diag first, then go through stencil neighbors */
col[diag].i = i; col[diag].j = j;
col[north].i = i; col[north].j = j+1;
col[east].i = i+1; col[east].j = j;
col[south].i = i; col[south].j = j-1;
col[west].i = i-1; col[west].j = j;
ierr = MatSetValuesStencil(A,1,&row,5,col,vals,INSERT_VALUES); CHKERRQ(ierr);
/* entries of vectors: exact solution u and right-hand-side b */
xi = (double)i * dx;
yj = (double)j * dy;
uarr[j][i] = sin(2.0 * pi * xi) * cos(4.0 * pi * yj);
barr[j][i] = (1.0 - 20.0 * pi * pi) * uarr[j][i];
barr[j][i] *= dx * dx;
}
}
DAVecRestoreArray(da2, b, &barr);
DAVecRestoreArray(da2, u, &uarr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = VecAssemblyBegin(b); CHKERRQ(ierr);
ierr = VecAssemblyEnd(b); CHKERRQ(ierr);
ierr = VecAssemblyBegin(u); CHKERRQ(ierr);
ierr = VecAssemblyEnd(u); CHKERRQ(ierr);
/* uncomment for dense view; -mat_view default format is good enough for most purposes
PetscViewer viewer;
PetscViewerCreate(com, &viewer);
PetscViewerSetType(viewer, PETSC_VIEWER_ASCII);
PetscViewerSetFormat(viewer, PETSC_VIEWER_ASCII_DENSE);
MatView(A,viewer);
PetscViewerDestroy(viewer);
*/
/* setup solver context now that Mat and Vec are assembled */
KSP ksp;
ierr = KSPCreate(com,&ksp);CHKERRQ(ierr);
/* Set "operators". Here the matrix that defines the linear system
also serves as the preconditioning matrix. But we do not assert a
relationship between their nonzero patterns.(???) */
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
/* Following is optional; parameters could be set at runtime. */
ierr = KSPSetTolerances(ksp,1.e-7,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
CHKERRQ(ierr);
/* Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* Solve linear system */
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* Compute and report the error (and the iteration count and reason). */
PetscScalar norminf, normtwo, neg_one=-1.0;
PetscInt its;
KSPConvergedReason reason;
ierr = VecAXPY(x,neg_one,u);CHKERRQ(ierr); // x = x - u
ierr = VecNorm(x,NORM_INFINITY,&norminf);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&normtwo);CHKERRQ(ierr); // discrete norm
normtwo *= dx * dy; // integral norm
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = KSPGetConvergedReason(ksp,&reason); CHKERRQ(ierr);
ierr = PetscPrintf(com,
"Error norms ||err||_inf = %.3e, ||err||_2 = %.3e;\n"
"Iterations = %d; Reason = %d\n",
norminf, normtwo, its, (int) reason);CHKERRQ(ierr);
/* destroy */
ierr = KSPDestroy(ksp);CHKERRQ(ierr);
ierr = MatDestroy(A);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(u);CHKERRQ(ierr);
ierr = VecDestroy(b);CHKERRQ(ierr);
ierr = DADestroy(da2);CHKERRQ(ierr);
/* Always call PetscFinalize() before exiting a program. */
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
