int main(int argc,char **argv)
{
DMMG *dmmg; /* multilevel grid structure */
PetscErrorCode ierr;
DA da;
AppCtx app;
PC pc;
KSP ksp;
PetscTruth isshell;
PetscViewer v1;
PetscInitialize(&argc,&argv,(char *)0,help);
PreLoadBegin(PETSC_TRUE,"SetUp");
app.comm = PETSC_COMM_WORLD;
app.nxv = 6;
app.nyvf = 3;
app.nyv = app.nyvf + 2;
ierr = PetscOptionsBegin(app.comm,PETSC_NULL,"Options for Grid Sizes",PETSC_NULL);
ierr = PetscOptionsInt("-nxv","Grid spacing in X direction",PETSC_NULL,app.nxv,&app.nxv,PETSC_NULL);
CHKERRQ(ierr);
ierr = PetscOptionsInt("-nyvf","Grid spacing in Y direction of Fuel",PETSC_NULL,app.nyvf,&app.nyvf,PETSC_NULL);
CHKERRQ(ierr);
ierr = PetscOptionsInt("-nyv","Total Grid spacing in Y direction of",PETSC_NULL,app.nyv,&app.nyv,PETSC_NULL);
CHKERRQ(ierr);
ierr = PetscOptionsEnd();
ierr = PetscViewerDrawOpen(app.comm,PETSC_NULL,"",-1,-1,-1,-1,&v1);
CHKERRQ(ierr);
/*
Create the DMComposite object to manage the three grids/physics.
We use a 1d decomposition along the y direction (since one of the grids is 1d).
*/
ierr = DMCompositeCreate(app.comm,&app.pack);
CHKERRQ(ierr);
/* 6 fluid unknowns, 3 ghost points on each end for either periodicity or simply boundary conditions */
ierr = DACreate1d(app.comm,DA_XPERIODIC,app.nxv,6,3,0,&da);
CHKERRQ(ierr);
ierr = DASetFieldName(da,0,"prss");
CHKERRQ(ierr);
ierr = DASetFieldName(da,1,"ergg");
CHKERRQ(ierr);
ierr = DASetFieldName(da,2,"ergf");
CHKERRQ(ierr);
ierr = DASetFieldName(da,3,"alfg");
CHKERRQ(ierr);
ierr = DASetFieldName(da,4,"velg");
CHKERRQ(ierr);
ierr = DASetFieldName(da,5,"velf");
CHKERRQ(ierr);
ierr = DMCompositeAddDM(app.pack,(DM)da);
CHKERRQ(ierr);
ierr = DADestroy(da);
CHKERRQ(ierr);
ierr = DACreate2d(app.comm,DA_YPERIODIC,DA_STENCIL_STAR,app.nxv,app.nyv,PETSC_DETERMINE,1,1,1,0,0,&da);
CHKERRQ(ierr);
ierr = DASetFieldName(da,0,"Tempature");
CHKERRQ(ierr);
ierr = DMCompositeAddDM(app.pack,(DM)da);
CHKERRQ(ierr);
ierr = DADestroy(da);
CHKERRQ(ierr);
ierr = DACreate2d(app.comm,DA_XYPERIODIC,DA_STENCIL_STAR,app.nxv,app.nyvf,PETSC_DETERMINE,1,2,1,0,0,&da);
CHKERRQ(ierr);
ierr = DASetFieldName(da,0,"Phi");
CHKERRQ(ierr);
ierr = DASetFieldName(da,1,"Pre");
CHKERRQ(ierr);
ierr = DMCompositeAddDM(app.pack,(DM)da);
CHKERRQ(ierr);
ierr = DADestroy(da);
CHKERRQ(ierr);
app.pri = 1.0135e+5;
app.ugi = 2.5065e+6;
app.ufi = 4.1894e+5;
app.agi = 1.00e-1;
app.vgi = 1.0e-1 ;
app.vfi = 1.0e-1;
app.prin = 1.0135e+5;
app.ugin = 2.5065e+6;
app.ufin = 4.1894e+5;
app.agin = 1.00e-1;
app.vgin = 1.0e-1 ;
app.vfin = 1.0e-1;
app.prout = 1.0135e+5;
app.ugout = 2.5065e+6;
app.ufout = 4.1894e+5;
app.agout = 3.0e-1;
app.twi = 373.15e+0;
app.phii = 1.0e+0;
app.prei = 1.0e-5;
/*
Create the solver object and attach the grid/physics info
*/
ierr = DMMGCreate(app.comm,1,0,&dmmg);
CHKERRQ(ierr);
ierr = DMMGSetDM(dmmg,(DM)app.pack);
CHKERRQ(ierr);
ierr = DMMGSetUser(dmmg,0,&app);
CHKERRQ(ierr);
ierr = DMMGSetISColoringType(dmmg,IS_COLORING_GLOBAL);
CHKERRQ(ierr);
CHKMEMQ;
ierr = DMMGSetInitialGuess(dmmg,FormInitialGuess);
CHKERRQ(ierr);
ierr = DMMGSetSNES(dmmg,FormFunction,0);
CHKERRQ(ierr);
ierr = DMMGSetFromOptions(dmmg);
CHKERRQ(ierr);
/* Supply custom shell preconditioner if requested */
ierr = SNESGetKSP(DMMGGetSNES(dmmg),&ksp);
CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);
CHKERRQ(ierr);
ierr = PetscTypeCompare((PetscObject)pc,PCSHELL,&isshell);
CHKERRQ(ierr);
if (isshell) {
ierr = PCShellSetContext(pc,&app);
CHKERRQ(ierr);
ierr = PCShellSetSetUp(pc,MyPCSetUp);
CHKERRQ(ierr);
ierr = PCShellSetApply(pc,MyPCApply);
CHKERRQ(ierr);
ierr = PCShellSetDestroy(pc,MyPCDestroy);
CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the nonlinear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
PreLoadStage("Solve");
ierr = DMMGSolve(dmmg);
CHKERRQ(ierr);
ierr = VecView(DMMGGetx(dmmg),v1);
CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscViewerDestroy(v1);
CHKERRQ(ierr);
ierr = DMCompositeDestroy(app.pack);
CHKERRQ(ierr);
ierr = DMMGDestroy(dmmg);
CHKERRQ(ierr);
PreLoadEnd();
ierr = PetscFinalize();
CHKERRQ(ierr);
return 0;
}
int main(int argc,char **args)
{
Vec u;
Mat A;
PetscErrorCode ierr;
mv_MultiVectorPtr eigenvectors;
PetscScalar * eigs;
PetscScalar * eigs_hist;
double * resid;
double * resid_hist;
int iterations;
PetscMPIInt rank;
int n_eigs = 1;
int seed = 1;
int i,j;
PetscLogDouble t1,t2,elapsed_time;
DA da;
double tol=1e-08;
PetscTruth option_present;
PetscTruth freepart=PETSC_FALSE;
PetscTruth full_output=PETSC_FALSE;
PetscInt m,n,p;
KSP ksp;
lobpcg_Tolerance lobpcg_tol;
int maxIt = 100;
mv_InterfaceInterpreter ii;
lobpcg_BLASLAPACKFunctions blap_fn;
aux_data_struct aux_data;
/*
PetscViewer viewer;
*/
PetscInt tmp_int;
mv_TempMultiVector * xe;
PetscInt N;
PetscScalar * xx;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n_eigs",&tmp_int,&option_present);CHKERRQ(ierr);
if (option_present)
n_eigs = tmp_int;
ierr = PetscOptionsGetReal(PETSC_NULL,"-tol", &tol,PETSC_NULL); CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-freepart",&freepart); CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-full_out",&full_output); CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-seed",&tmp_int,&option_present);CHKERRQ(ierr);
if (option_present)
seed = tmp_int;
ierr = PetscOptionsGetInt(PETSC_NULL,"-itr",&tmp_int,&option_present);CHKERRQ(ierr);
if (option_present)
maxIt = tmp_int;
if (seed<1)
seed=1;
/* we actually run our code twice: first time we solve small problem just to make sure
that all program code is actually loaded into memory; then we solve the problem
we are interested in; this trick is done for accurate timing
*/
PreLoadBegin(PETSC_TRUE,"grid and matrix assembly");
/* "create" the grid and stencil data; on first run we form small problem */
if (PreLoadIt==0)
{
/* small problem */
ierr=DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,10,10,10,
1,PETSC_DECIDE,1,1,1,0,0,0,&da); CHKERRQ(ierr);
}
else
{
/* actual problem */
if (freepart) /* petsc determines partitioning */
{
ierr=DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,-10,-10,-10,
PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0,0,&da); CHKERRQ(ierr);
}
else /* (1,NP,1) partitioning */
{
ierr=DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,-10,-10,-10,
1,PETSC_DECIDE,1,1,1,0,0,0,&da); CHKERRQ(ierr);
}
/* now we print what partitioning is chosen */
ierr=DAGetInfo(da,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL,&m,
&n,&p,PETSC_NULL,PETSC_NULL,PETSC_NULL,PETSC_NULL); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Partitioning: %u %u %u\n",m,n,p);
}
/* create matrix, whose nonzero structure and probably partitioning corresponds to
grid and stencil structure */
ierr=DAGetMatrix(da,MATMPIAIJ,&A); CHKERRQ(ierr);
/* fill the matrix with values. I intend to build 7-pt Laplas */
/* this procedure includes matrix assembly */
ierr=FillMatrix(da,A); CHKERRQ(ierr);
/*
PetscViewerBinaryOpen(PETSC_COMM_WORLD,"matrix.dat",FILE_MODE_WRITE,&viewer);
MatView(A,PETSC_VIEWER_STDOUT_WORLD);
PetscViewerDestroy(viewer);
*/
/*
Create parallel vectors.
- We form 1 vector from scratch and then duplicate as needed.
*/
ierr = DACreateGlobalVector(da,&u); CHKERRQ(ierr);
/* ierr = VecSetFromOptions(u);CHKERRQ(ierr); */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the linear solver and set various options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* Here we START measuring time for preconditioner setup */
PreLoadStage("preconditioner setup");
ierr = PetscGetTime(&t1);CHKERRQ(ierr);
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
/*
Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
These options will override those specified above as long as
KSPSetFromOptions() is called _after_ any other customization
routines.
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* probably this call actually builds the preconditioner */
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
/* Here we STOP measuring time for preconditioner setup */
PreLoadStage("solution");
ierr = PetscGetTime(&t2);CHKERRQ(ierr);
elapsed_time=t2-t1;
if (PreLoadIt==1)
PetscPrintf(PETSC_COMM_WORLD,"Preconditioner setup, seconds: %f\n",elapsed_time);
/* request memory for eig-vals */
ierr = PetscMalloc(sizeof(PetscScalar)*n_eigs,&eigs); CHKERRQ(ierr);
/* request memory for eig-vals history */
ierr = PetscMalloc(sizeof(PetscScalar)*n_eigs*(maxIt+1),&eigs_hist); CHKERRQ(ierr);
/* request memory for resid. norms */
ierr = PetscMalloc(sizeof(double)*n_eigs,&resid); CHKERRQ(ierr);
/* request memory for resid. norms hist. */
ierr = PetscMalloc(sizeof(double)*n_eigs*(maxIt+1),&resid_hist); CHKERRQ(ierr);
LOBPCG_InitRandomContext();
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
PETSCSetupInterpreter( &ii );
eigenvectors = mv_MultiVectorCreateFromSampleVector(&ii, n_eigs,u);
xe = (mv_TempMultiVector *) mv_MultiVectorGetData( eigenvectors );
/*
VecView( (Vec)xe->vector[0],PETSC_VIEWER_STDOUT_WORLD);
*/
for (i=0; i<seed; i++) /* this cycle is to imitate changing random seed */
mv_MultiVectorSetRandom (eigenvectors, 1234);
/*
VecView( (Vec)xe->vector[0],PETSC_VIEWER_STDOUT_WORLD);
*/
VecGetSize( (Vec)xe->vector[0], &N );
N=mv_TempMultiVectorHeight( xe );
VecGetArray( (Vec)xe->vector[0],&xx);
lobpcg_tol.absolute = tol;
lobpcg_tol.relative = 1e-50;
#ifdef PETSC_USE_COMPLEX
blap_fn.zpotrf = PETSC_zpotrf_interface;
blap_fn.zhegv = PETSC_zsygv_interface;
#else
blap_fn.dpotrf = PETSC_dpotrf_interface;
blap_fn.dsygv = PETSC_dsygv_interface;
#endif
aux_data.A = A;
aux_data.ksp = ksp;
aux_data.ii = ii;
/* Here we START measuring time for solution process */
ierr = PetscGetTime(&t1);CHKERRQ(ierr);
#ifdef PETSC_USE_COMPLEX
lobpcg_solve_complex(
eigenvectors, /*input-initial guess of e-vectors */
&aux_data, /*input-matrix A */
OperatorAMultiVector, /*input-operator A */
NULL, /*input-matrix B */
NULL, /*input-operator B */
&aux_data, /*input-matrix T */
Precond_FnMultiVector, /*input-operator T */
NULL, /*input-matrix Y */
blap_fn, /*input-lapack functions */
lobpcg_tol, /*input-tolerances */
PreLoadIt? maxIt:1, /*input-max iterations */
!rank && PreLoadIt, /*input-verbosity level */
&iterations, /*output-actual iterations */
(komplex *) eigs, /*output-eigenvalues */
(komplex *) eigs_hist, /*output-eigenvalues history */
n_eigs, /*output-history global height */
resid, /*output-residual norms */
resid_hist , /*output-residual norms history */
n_eigs /*output-history global height */
);
#else
lobpcg_solve_double( eigenvectors,
&aux_data,
OperatorAMultiVector,
NULL,
NULL,
&aux_data,
Precond_FnMultiVector,
NULL,
blap_fn,
lobpcg_tol,
PreLoadIt? maxIt:1,
!rank && PreLoadIt,
&iterations,
eigs, /* eigenvalues; "lambda_values" should point to array
containing <blocksize> doubles where <blocksize> is the
width of multivector "blockVectorX" */
eigs_hist,
/* eigenvalues history; a pointer to the entries of the
<blocksize>-by-(<maxIterations>+1) matrix stored in
fortran-style. (i.e. column-wise) The matrix may be
a submatrix of a larger matrix, see next argument */
n_eigs, /* global height of the matrix (stored in fotran-style)
specified by previous argument */
resid,
/* residual norms; argument should point to
array of <blocksize> doubles */
resid_hist ,
/* residual norms history; a pointer to the entries of the
<blocksize>-by-(<maxIterations>+1) matrix stored in
fortran-style. (i.e. column-wise) The matrix may be
a submatrix of a larger matrix, see next argument */
n_eigs
/* global height of the matrix (stored in fotran-style)
specified by previous argument */
);
#endif
/* Here we STOP measuring time for solution process */
ierr = PetscGetTime(&t2);CHKERRQ(ierr);
elapsed_time=t2-t1;
if (PreLoadIt)
PetscPrintf(PETSC_COMM_WORLD,"Solution process, seconds: %e\n",elapsed_time);
if (PreLoadIt && full_output)
{
PetscPrintf(PETSC_COMM_WORLD,"Output from LOBPCG, eigenvalues:\n");
for (i=0;i<n_eigs;i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD,"%e\n",PetscRealPart(eigs[i]));
CHKERRQ(ierr);
}
PetscPrintf(PETSC_COMM_WORLD,"Output from LOBPCG, eigenvalues history:\n");
for (j=0; j<iterations+1; j++)
for (i=0;i<n_eigs;i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD,"%e\n",PetscRealPart(*(eigs_hist+j*n_eigs+i)));
CHKERRQ(ierr);
}
PetscPrintf(PETSC_COMM_WORLD,"Output from LOBPCG, residual norms:\n");
for (i=0;i<n_eigs;i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD,"%e\n",resid[i]);
CHKERRQ(ierr);
}
PetscPrintf(PETSC_COMM_WORLD,"Output from LOBPCG, residual norms history:\n");
for (j=0; j<iterations+1; j++)
for (i=0;i<n_eigs;i++)
{
ierr = PetscPrintf(PETSC_COMM_WORLD,"%e\n",*(resid_hist+j*n_eigs+i));
CHKERRQ(ierr);
}
}
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = VecDestroy(u);CHKERRQ(ierr);
ierr = MatDestroy(A);CHKERRQ(ierr);
ierr = KSPDestroy(ksp);CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
LOBPCG_DestroyRandomContext();
mv_MultiVectorDestroy(eigenvectors);
/* free memory used for eig-vals */
ierr = PetscFree(eigs); CHKERRQ(ierr);
ierr = PetscFree(eigs_hist); CHKERRQ(ierr);
ierr = PetscFree(resid); CHKERRQ(ierr);
ierr = PetscFree(resid_hist); CHKERRQ(ierr);
/*
Always call PetscFinalize() before exiting a program. This routine
- finalizes the PETSc libraries as well as MPI
- provides summary and diagnostic information if certain runtime
options are chosen (e.g., -log_summary).
*/
PreLoadEnd();
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
