void
petscSetOptions(FEProblem & problem)
{
// Reference to the options stored in FEPRoblem
PetscOptions & petsc = problem.getPetscOptions();
if (petsc.inames.size() != petsc.values.size())
mooseError("PETSc names and options are not the same length");
PetscOptionsClear();
{ // Get any options specified on the command-line
int argc;
char ** args;
PetscGetArgs(&argc, &args);
PetscOptionsInsert(&argc, &args, NULL);
}
setSolverOptions(problem.solverParams());
// Add any additional options specified in the input file
for (MooseEnumIterator it = petsc.flags.begin(); it != petsc.flags.end(); ++it)
PetscOptionsSetValue(it->c_str(), PETSC_NULL);
for (unsigned int i=0; i<petsc.inames.size(); ++i)
PetscOptionsSetValue(petsc.inames[i].c_str(), petsc.values[i].c_str());
SolverParams& solver_params = problem.solverParams();
if (solver_params._type != Moose::ST_JFNK &&
solver_params._type != Moose::ST_FD &&
!problem.getNonlinearSystem().haveFiniteDifferencedPreconditioner() &&
problem.getNonlinearSystem().haveDecomposition())
{
// Set up DM only if have a decomposition. Additionally, turn DM OFF if not using FD-based solvers,
// (both -snes_mf and -snes_fd) and FDP. This is all rather crufty, but what's a good generic rule here?
// In principle at least, splits should be able to work with ST_FD (-snes_fd) and FDP (a coloring-based
// version of -snes_fd), but one has to be careful about the initialization order so as not to override
// SNESComputeJacobianDefaultColor() set up by FDP, for instance. However, it's unlikely that splits
// will be used when running an FD solver (debugging).
problem.getNonlinearSystem().setupDecomposition();
petscSetupDM(problem.getNonlinearSystem());
} else {
// Otherwise turn off the decomposition
std::vector<std::string> nosplits;
problem.getNonlinearSystem().setDecomposition(nosplits);
}
}
static PetscErrorCode TaoPounders(AppCtx *user)
{
PetscErrorCode ierr;
Tao tao;
Vec X, F;
Mat J;
char buf[1024];
PetscFunctionBegin;
/* Set the values for the algorithm options we want to use */
sprintf(buf,"%d",user->nfmax);
ierr = PetscOptionsSetValue(NULL,"-tao_max_funcs",buf);CHKERRQ(ierr);
sprintf(buf,"%d",user->npmax);
ierr = PetscOptionsSetValue(NULL,"-tao_pounders_npmax",buf);CHKERRQ(ierr);
sprintf(buf,"%5.4e",user->delta);
ierr = PetscOptionsSetValue(NULL,"-tao_pounders_delta",buf);CHKERRQ(ierr);
/* Create the TAO objects and set the type */
ierr = TaoCreate(PETSC_COMM_SELF,&tao);CHKERRQ(ierr);
/* Create starting point and initialize */
ierr = VecCreateSeq(PETSC_COMM_SELF,user->n,&X);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)X,"X0");CHKERRQ(ierr);
ierr = PetscMatlabEngineGet(user->mengine,(PetscObject)X);CHKERRQ(ierr);
ierr = TaoSetInitialVector(tao,X);CHKERRQ(ierr);
/* Create residuals vector and set residual function */
ierr = VecCreateSeq(PETSC_COMM_SELF,user->m,&F);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)F,"F");CHKERRQ(ierr);
ierr = TaoSetResidualRoutine(tao,F,EvaluateResidual,(void*)user);CHKERRQ(ierr);
/* Create Jacobian matrix and set residual Jacobian routine */
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,user->m,user->n,user->n,NULL,&J);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)J,"J");CHKERRQ(ierr);
ierr = TaoSetResidualJacobianRoutine(tao,J,J,EvaluateJacobian,(void*)user);CHKERRQ(ierr);
/* Solve the problem */
ierr = TaoSetType(tao,TAOPOUNDERS);CHKERRQ(ierr);
ierr = TaoSetFromOptions(tao);CHKERRQ(ierr);
ierr = TaoSolve(tao);CHKERRQ(ierr);
/* Finish the problem */
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&F);CHKERRQ(ierr);
ierr = TaoDestroy(&tao);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
PetscErrorCode ierr;
Vec U,cv,eta;
DM da,da2;
PetscViewer viewer,view_vtk_cv,view_vtk_eta;
char filename[PETSC_MAX_PATH_LEN],cv_filename[PETSC_MAX_PATH_LEN],eta_filename[PETSC_MAX_PATH_LEN];
PetscBool flg,sflg = PETSC_FALSE;
PetscInt i,n=10000;
PetscInt seed;
PetscInitialize(&argc,&argv, (char*)0, help);
ierr = PetscOptionsSetValue("-viewer_binary_skip_info","true");CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-f",filename,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) {
ierr = PetscOptionsGetInt(PETSC_NULL,"-random_seed",&seed,&sflg);CHKERRQ(ierr);
if (!sflg) {
ierr = PetscStrcpy(filename,"ex61.data");CHKERRQ(ierr);
} else {
sprintf(filename,"ex61.data.%d",seed);
}
}
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,filename,FILE_MODE_READ,&viewer);CHKERRQ(ierr);
/* Get physics and time parameters */
ierr = DMCreate(PETSC_COMM_WORLD,&da);CHKERRQ(ierr);
ierr = DMLoad(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&U);CHKERRQ(ierr);
ierr = DMDAGetReducedDA(da,1,&da2);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da2,&cv);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da2,&eta);CHKERRQ(ierr);
for (i=0; i<n; i++) {
/* when this fails it simply means the file is finished */
ierr = VecLoad(U,viewer);CHKERRQ(ierr);
ierr = VecStrideGather(U,1,cv,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecStrideGather(U,4,eta,INSERT_VALUES);CHKERRQ(ierr);
sprintf(cv_filename,"%s_cv_%d.vtk",filename,i);
sprintf(eta_filename,"%s_eta_%d.vtk",filename,i);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,cv_filename,&view_vtk_cv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,eta_filename,&view_vtk_eta);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(view_vtk_cv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(view_vtk_eta, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(da2,view_vtk_cv);CHKERRQ(ierr);
ierr = DMView(da2,view_vtk_eta);CHKERRQ(ierr);
ierr = VecView(cv,view_vtk_cv);CHKERRQ(ierr);
ierr = VecView(eta,view_vtk_eta);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&view_vtk_cv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&view_vtk_eta);CHKERRQ(ierr);
}
ierr = VecDestroy(&U);CHKERRQ(ierr);
ierr = VecDestroy(&cv);CHKERRQ(ierr);
ierr = VecDestroy(&eta);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = DMDestroy(&da2);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
static PetscErrorCode KSPComputeShifts_GMRES(KSP ksp)
{
PetscErrorCode ierr;
KSP_AGMRES *agmres = (KSP_AGMRES*)(ksp->data);
KSP kspgmres;
Mat Amat, Pmat;
MatStructure flag;
PetscInt max_k = agmres->max_k;
PC pc;
PetscInt m;
PetscScalar *Rshift, *Ishift;
PetscFunctionBegin;
/* Perform one cycle of classical GMRES (with the Arnoldi process) to get the Hessenberg matrix
We assume here that the ksp is AGMRES and that the operators for the
linear system have been set in this ksp */
ierr = KSPCreate(PetscObjectComm((PetscObject)ksp), &kspgmres);CHKERRQ(ierr);
if (!ksp->pc) { ierr = KSPGetPC(ksp,&ksp->pc);CHKERRQ(ierr); }
ierr = PCGetOperators(ksp->pc, &Amat, &Pmat);CHKERRQ(ierr);
ierr = KSPSetOperators(kspgmres, Amat, Pmat);CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspgmres);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL, "-ksp_view", &flg);CHKERRQ(ierr);
if (flag) { ierr = PetscOptionsClearValue("-ksp_view");CHKERRQ(ierr); }
ierr = KSPSetType(kspgmres, KSPGMRES);CHKERRQ(ierr);
ierr = KSPGMRESSetRestart(kspgmres, max_k);CHKERRQ(ierr);
ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr);
ierr = KSPSetPC(kspgmres, pc);CHKERRQ(ierr);
/* Copy common options */
kspgmres->pc_side = ksp->pc_side;
/* Setup KSP context */
ierr = KSPSetComputeEigenvalues(kspgmres, PETSC_TRUE);CHKERRQ(ierr);
ierr = KSPSetUp(kspgmres);CHKERRQ(ierr);
kspgmres->max_it = max_k; /* Restrict the maximum number of iterations to one cycle of GMRES */
kspgmres->rtol = ksp->rtol;
ierr = KSPSolve(kspgmres, ksp->vec_rhs, ksp->vec_sol);CHKERRQ(ierr);
ksp->guess_zero = PETSC_FALSE;
ksp->rnorm = kspgmres->rnorm;
ksp->its = kspgmres->its;
if (kspgmres->reason == KSP_CONVERGED_RTOL) {
ksp->reason = KSP_CONVERGED_RTOL;
PetscFunctionReturn(0);
} else ksp->reason = KSP_CONVERGED_ITERATING;
/* Now, compute the Shifts values */
ierr = PetscMalloc2(max_k,&Rshift,max_k,&Ishift);CHKERRQ(ierr);
ierr = KSPComputeEigenvalues(kspgmres, max_k, Rshift, Ishift, &m);CHKERRQ(ierr);
if (m < max_k) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_PLIB, "Unable to compute the Shifts for the Newton basis");
else {
ierr = KSPAGMRESLejaOrdering(Rshift, Ishift, agmres->Rshift, agmres->Ishift, max_k);CHKERRQ(ierr);
agmres->HasShifts = PETSC_TRUE;
}
/* Restore KSP view options */
if (flg) { ierr = PetscOptionsSetValue("-ksp_view", "");CHKERRQ(ierr); }
PetscFunctionReturn(0);
}
int main(int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start\n"); CHKERRQ(ierr);
int d1 = 5;
ierr = PetscOptionsSetValue("-da_grid_x","5"); CHKERRQ(ierr);
int d2 = 7;
ierr = PetscOptionsSetValue("-da_grid_y","7"); CHKERRQ(ierr);
FluidField f;
ierr = FluidFieldCreate(&f); CHKERRQ(ierr);
iCoor s = {d1,d2,0};
Grid g;
ierr = GridCreate(s,&g); CHKERRQ(ierr);
PetscReal *p;
VecGetArray(f->p, &p);
for (int i = 0; i < g->n.x*g->n.y; ++i)
{
g->v1[i] = i;
p[i] = i;
}
PetscReal **pp;
DAVecGetArray(f->da, f->p, &pp);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
printf( "%1.0f, %1.0f\t", g->v2[j][i], pp[j][i]);
}
printf("\n");
}
DAVecRestoreArray(f->da, f->p, &pp);
ierr = GridDestroy(g); CHKERRQ(ierr);
ierr = FluidFieldDestroy(f); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "End\n"); CHKERRQ(ierr);
ierr = PetscFinalize(); CHKERRQ(ierr);
return 0;
}
void
setSinglePetscOption(const std::string & name, const std::string & value)
{
PetscErrorCode ierr;
#if PETSC_VERSION_LESS_THAN(3,7,0)
ierr = PetscOptionsSetValue(name.c_str(), value == "" ? PETSC_NULL : value.c_str());
#else
// PETSc 3.7.0 and later version. First argument is the options
// database to use, NULL indicates the default global database.
ierr = PetscOptionsSetValue(PETSC_NULL, name.c_str(), value == "" ? PETSC_NULL : value.c_str());
#endif
// Not convenient to use the usual error checking macro, because we
// don't have a specific communicator in this helper function.
if (ierr)
mooseError("Error setting PETSc option.");
}
PetscErrorCode RunTest(int nx, int ny, int nz, int loops, double *wt)
{
Vec x,f;
TS ts;
AppCtx _app,*app=&_app;
double t1,t2;
PetscErrorCode ierr;
PetscFunctionBegin;
app->nx = nx; app->h[0] = 1./(nx-1);
app->ny = ny; app->h[1] = 1./(ny-1);
app->nz = nz; app->h[2] = 1./(nz-1);
ierr = VecCreate(PETSC_COMM_SELF,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,nx*ny*nz,nx*ny*nz);CHKERRQ(ierr);
ierr = VecSetUp(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&f);CHKERRQ(ierr);
ierr = TSCreate(PETSC_COMM_SELF,&ts);CHKERRQ(ierr);
ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr);
ierr = TSSetType(ts,TSTHETA);CHKERRQ(ierr);
ierr = TSThetaSetTheta(ts,1.0);CHKERRQ(ierr);
ierr = TSSetTimeStep(ts,0.01);CHKERRQ(ierr);
ierr = TSSetTime(ts,0.0);CHKERRQ(ierr);
ierr = TSSetDuration(ts,10,1.0);CHKERRQ(ierr);
ierr = TSSetSolution(ts,x);CHKERRQ(ierr);
ierr = TSSetIFunction(ts,f,FormFunction,app);CHKERRQ(ierr);
ierr = PetscOptionsSetValue("-snes_mf","1");CHKERRQ(ierr);
{
SNES snes;
KSP ksp;
ierr = TSGetSNES(ts,&snes);CHKERRQ(ierr);
ierr = SNESGetKSP(snes,&ksp);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPCG);CHKERRQ(ierr);
}
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
ierr = TSSetUp(ts);CHKERRQ(ierr);
*wt = 1e300;
while (loops-- > 0) {
ierr = FormInitial(0.0,x,app);CHKERRQ(ierr);
ierr = PetscGetTime(&t1);CHKERRQ(ierr);
ierr = TSSolve(ts,x,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscGetTime(&t2);CHKERRQ(ierr);
*wt = PetscMin(*wt,t2-t1);
}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&f);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
PetscInt opts[6] = {0};
PetscBool hascl = PETSC_FALSE,hasstr = PETSC_FALSE;
ierr = PetscInitialize(&argc,&argv,0,help);CHKERRQ(ierr);
ierr = PetscOptionsSetValue(NULL,"-zero","0");CHKERRQ(ierr);
ierr = PetscOptionsPrefixPush(NULL,"a_");CHKERRQ(ierr);
ierr = PetscOptionsSetValue(NULL,"-one","1");CHKERRQ(ierr);
ierr = PetscOptionsPrefixPush(NULL,"bb_");CHKERRQ(ierr);
ierr = PetscOptionsSetValue(NULL,"-two","2");CHKERRQ(ierr);
ierr = PetscOptionsPrefixPop(NULL);CHKERRQ(ierr);
ierr = PetscOptionsSetValue(NULL,"-three","3");CHKERRQ(ierr);
ierr = PetscOptionsPrefixPush(NULL,"cc_");CHKERRQ(ierr);
ierr = PetscOptionsPrefixPush(NULL,"ddd_");CHKERRQ(ierr);
ierr = PetscOptionsSetValue(NULL,"-four","4");CHKERRQ(ierr);
ierr = PetscOptionsPrefixPop(NULL);CHKERRQ(ierr);
ierr = PetscOptionsPrefixPop(NULL);CHKERRQ(ierr);
ierr = PetscOptionsPrefixPop(NULL);CHKERRQ(ierr);
ierr = PetscOptionsSetValue(NULL,"-five","5");CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-zero",&opts[0],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-a_one",&opts[1],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-a_bb_two",&opts[2],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-a_three",&opts[3],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-a_cc_ddd_four",&opts[4],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-five",&opts[5],0);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"opts = {%D %D %D %D %D %D}\n",opts[0],opts[1],opts[2],opts[3],opts[4],opts[5]);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,0,"-cl",&hascl,0);CHKERRQ(ierr);
if (hascl) {
ierr = PetscMemzero(opts,sizeof(opts));CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-cl_zero",&opts[0],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-cl_a_one",&opts[1],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-cl_a_bb_two",&opts[2],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-cl_a_three",&opts[3],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-cl_a_cc_ddd_four",&opts[4],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-cl_five",&opts[5],0);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"cl_opts = {%D %D %D %D %D %D}\n",opts[0],opts[1],opts[2],opts[3],opts[4],opts[5]);CHKERRQ(ierr);
}
ierr = PetscOptionsGetBool(NULL,0,"-str",&hasstr,0);CHKERRQ(ierr);
if (hasstr) {
ierr = PetscOptionsInsertString(NULL,"-prefix_push str_ -zero 100 -prefix_push a_ -one 101 -prefix_push bb_ -two 102 -prefix_pop -three 103 -prefix_push cc_ -prefix_push ddd_ -four 104 -prefix_pop -prefix_pop -prefix_pop -five 105 -prefix_pop");CHKERRQ(ierr);
ierr = PetscMemzero(opts,sizeof(opts));CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-str_zero",&opts[0],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-str_a_one",&opts[1],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-str_a_bb_two",&opts[2],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-str_a_three",&opts[3],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-str_a_cc_ddd_four",&opts[4],0);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,0,"-str_five",&opts[5],0);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"str_opts = {%D %D %D %D %D %D}\n",opts[0],opts[1],opts[2],opts[3],opts[4],opts[5]);CHKERRQ(ierr);
}
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
void
ContactSplit::setup(const std::string& prefix)
{
PetscErrorCode ierr;
std::string dmprefix = prefix+"dm_moose_", opt, val;
// contacts options
if (_contact_master.size()) {
opt = dmprefix+"ncontacts";
{
std::ostringstream oval;
oval << _contact_master.size();
val = oval.str();
}
ierr = PetscOptionsSetValue(opt.c_str(),val.c_str());
CHKERRABORT(_communicator.get(),ierr);
for (unsigned int j = 0; j < _contact_master.size(); ++j) {
std::ostringstream oopt;
oopt << dmprefix << "contact_" << j;
opt = oopt.str();
val = _contact_master[j]+","+_contact_slave[j];
ierr = PetscOptionsSetValue(opt.c_str(),val.c_str());
CHKERRABORT(_communicator.get(),ierr);
if (_contact_displaced[j]) {
opt = opt + "_displaced";
val = "yes";
ierr = PetscOptionsSetValue(opt.c_str(),val.c_str());
CHKERRABORT(_communicator.get(),ierr);
}
}
}
// uncontacts options
if (_uncontact_master.size()) {
opt = dmprefix+"nuncontacts";
{
std::ostringstream oval;
oval << _uncontact_master.size();
val = oval.str();
}
ierr = PetscOptionsSetValue(opt.c_str(),val.c_str());
CHKERRABORT(_communicator.get(),ierr);
for (unsigned int j = 0; j < _uncontact_master.size(); ++j) {
std::ostringstream oopt;
oopt << dmprefix << "uncontact_" << j;
opt = oopt.str();
val = _uncontact_master[j]+","+_uncontact_slave[j];
ierr = PetscOptionsSetValue(opt.c_str(),val.c_str());
CHKERRABORT(_communicator.get(),ierr);
if (_uncontact_displaced[j]) {
opt = opt + "_displaced";
val = "yes";
ierr = PetscOptionsSetValue(opt.c_str(),val.c_str());
CHKERRABORT(_communicator.get(),ierr);
}
}
}
Split::setup(prefix);
}
/*
* This is a modified version of PETSc/src/ts/examples/tutorials/ex15.c
* to demonstrate how MOOSE interact with an external solver package
*/
PetscErrorCode
externalPETScDiffusionFDMSolve(TS ts, Vec u, PetscReal dt, PetscReal time)
{
PetscErrorCode ierr;
#if !PETSC_VERSION_LESS_THAN(3, 8, 0)
PetscInt current_step;
#endif
DM da;
PetscFunctionBeginUser;
ierr = TSGetDM(ts, &da);
CHKERRQ(ierr);
#if !PETSC_VERSION_LESS_THAN(3, 7, 0)
PetscOptionsSetValue(NULL, "-ts_monitor", NULL);
PetscOptionsSetValue(NULL, "-snes_monitor", NULL);
PetscOptionsSetValue(NULL, "-ksp_monitor", NULL);
#else
PetscOptionsSetValue("-ts_monitor", NULL);
PetscOptionsSetValue("-snes_monitor", NULL);
PetscOptionsSetValue("-ksp_monitor", NULL);
#endif
/*ierr = TSSetMaxTime(ts,1.0);CHKERRQ(ierr);*/
ierr = TSSetExactFinalTime(ts, TS_EXACTFINALTIME_STEPOVER);
CHKERRQ(ierr);
ierr = TSSetSolution(ts, u);
CHKERRQ(ierr);
ierr = TSSetTimeStep(ts, dt);
CHKERRQ(ierr);
ierr = TSSetTime(ts, time - dt);
CHKERRQ(ierr);
#if !PETSC_VERSION_LESS_THAN(3, 8, 0)
ierr = TSGetStepNumber(ts, ¤t_step);
CHKERRQ(ierr);
ierr = TSSetMaxSteps(ts, current_step + 1);
CHKERRQ(ierr);
#else
SETERRQ(PetscObjectComm((PetscObject)ts), PETSC_ERR_SUP, "Require PETSc-3.8.x or higher ");
#endif
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Sets various TS parameters from user options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSSetFromOptions(ts);
CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve nonlinear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSSolve(ts, u);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void TestPerf() throw(Exception)
{
// solver and preconditioner options
HeartConfig::Instance()->SetKSPSolver("symmlq");
HeartConfig::Instance()->SetKSPPreconditioner("bjacobi");
PetscOptionsSetValue("-log_summary", "");
// write headings
PerformanceTester<CellLuoRudy1991FromCellMLBackwardEuler, BidomainProblem<3>, 3>::DisplayHeadings();
HeartEventHandler::Headings();
// base line
PerformanceTester<CellLuoRudy1991FromCellMLBackwardEuler, BidomainProblem<3>, 3> tester;
tester.MeshNum=4;
tester.Run();
HeartEventHandler::Report();
}
void
setSolverOptions(SolverParams & solver_params)
{
// set PETSc options implied by a solve type
switch (solver_params._type)
{
case Moose::ST_PJFNK:
PetscOptionsSetValue("-snes_mf_operator", PETSC_NULL);
break;
case Moose::ST_JFNK:
PetscOptionsSetValue("-snes_mf", PETSC_NULL);
break;
case Moose::ST_NEWTON:
break;
case Moose::ST_FD:
PetscOptionsSetValue("-snes_fd", PETSC_NULL);
break;
case Moose::ST_LINEAR:
PetscOptionsSetValue("-snes_type", "ksponly");
break;
}
Moose::LineSearchType ls_type = solver_params._line_search;
if (ls_type == Moose::LS_NONE)
ls_type = Moose::LS_BASIC;
if (ls_type != Moose::LS_DEFAULT)
{
#if PETSC_VERSION_LESS_THAN(3,3,0)
PetscOptionsSetValue("-snes_type", "ls");
PetscOptionsSetValue("-snes_ls", stringify(ls_type));
#else
PetscOptionsSetValue("-snes_linesearch_type", stringify(ls_type).c_str());
#endif
}
}
/*! \brief solve simple use a Krylov solver + Simple selected Parallel Pre-conditioner
*
*/
void solve_Krylov_simple(Mat & A_, const Vec & b_, Vec & x_)
{
PETSC_SAFE_CALL(KSPSetType(ksp,s_type));
// We set the size of x according to the Matrix A
PetscInt row;
PetscInt col;
PetscInt row_loc;
PetscInt col_loc;
PETSC_SAFE_CALL(MatGetSize(A_,&row,&col));
PETSC_SAFE_CALL(MatGetLocalSize(A_,&row_loc,&col_loc));
PC pc;
// We set the Matrix operators
PETSC_SAFE_CALL(KSPSetOperators(ksp,A_,A_));
// We set the pre-conditioner
PETSC_SAFE_CALL(KSPGetPC(ksp,&pc));
// PETSC_SAFE_CALL(PCSetType(pc,PCJACOBI));
PETSC_SAFE_CALL(PCSetType(pc,PCHYPRE));
// PCGAMGSetNSmooths(pc,0);
PCFactorSetShiftType(pc, MAT_SHIFT_NONZERO);
PCFactorSetShiftAmount(pc, PETSC_DECIDE);
PCHYPRESetType(pc, "boomeramg");
MatSetBlockSize(A_,4);
PetscOptionsSetValue("-pc_hypre_boomeramg_print_statistics","2");
PetscOptionsSetValue("-pc_hypre_boomeramg_max_iter","1000");
PetscOptionsSetValue("-pc_hypre_boomeramg_nodal_coarsen","true");
PetscOptionsSetValue("-pc_hypre_boomeramg_relax_type_all","SOR/Jacobi");
PetscOptionsSetValue("-pc_hypre_boomeramg_coarsen_type","Falgout");
PetscOptionsSetValue("-pc_hypre_boomeramg_cycle_type","W");
PetscOptionsSetValue("-pc_hypre_boomeramg_max_levels","10");
KSPSetFromOptions(ksp);
// if we are on on best solve set-up a monitor function
if (try_solve == true)
{
// for bench-mark we are interested in non-preconditioned norm
PETSC_SAFE_CALL(KSPMonitorSet(ksp,monitor,&vres,NULL));
// Disable convergence check
PETSC_SAFE_CALL(KSPSetConvergenceTest(ksp,KSPConvergedSkip,NULL,NULL));
}
// Solve the system
PETSC_SAFE_CALL(KSPSolve(ksp,b_,x_));
auto & v_cl = create_vcluster();
// if (try_solve == true)
// {
// calculate error statistic about the solution
solError err = statSolutionError(A_,b_,x_);
if (v_cl.getProcessUnitID() == 0)
{
std::cout << "Method: " << s_type << " " << " pre-conditoner: " << PCJACOBI << " iterations: " << err.its << std::endl;
std::cout << "Norm of error: " << err.err_norm << " Norm infinity: " << err.err_inf << std::endl;
}
// }
}
int main(int argc,char **argv)
{
PetscMPIInt rank,size;
PetscInt N = 6,M=8,P=5,dof=1;
PetscInt stencil_width=1,pt=0,st=0;
PetscErrorCode ierr;
PetscBool flg2,flg3,isbinary,mpiio;
DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE,bz = DM_BOUNDARY_NONE;
DMDAStencilType stencil_type = DMDA_STENCIL_STAR;
DM da,da2;
Vec global1,global2;
PetscScalar mone = -1.0;
PetscReal norm;
PetscViewer viewer;
PetscRandom rdm;
#if defined(PETSC_HAVE_HDF5)
PetscBool ishdf5;
#endif
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-P",&P,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-dof",&dof,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-stencil_width",&stencil_width,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-periodic",&pt,NULL);CHKERRQ(ierr);
if (pt == 1) bx = DM_BOUNDARY_PERIODIC;
if (pt == 2) by = DM_BOUNDARY_PERIODIC;
if (pt == 4) {bx = DM_BOUNDARY_PERIODIC; by = DM_BOUNDARY_PERIODIC;}
ierr = PetscOptionsGetInt(NULL,NULL,"-stencil_type",&st,NULL);CHKERRQ(ierr);
stencil_type = (DMDAStencilType) st;
ierr = PetscOptionsHasName(NULL,NULL,"-oned",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL,"-twod",&flg2);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL,"-threed",&flg3);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL,"-binary",&isbinary);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HDF5)
ierr = PetscOptionsHasName(NULL,NULL,"-hdf5",&ishdf5);CHKERRQ(ierr);
#endif
ierr = PetscOptionsHasName(NULL,NULL,"-mpiio",&mpiio);CHKERRQ(ierr);
if (flg2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,stencil_type,M,N,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,&da);CHKERRQ(ierr);
} else if (flg3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,0,&da);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,dof,stencil_width,NULL,&da);CHKERRQ(ierr);
}
ierr = DMCreateGlobalVector(da,&global1);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)global1,"Test_Vec");CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = VecSetRandom(global1,rdm);CHKERRQ(ierr);
if (isbinary) {
if (mpiio) {
ierr = PetscOptionsSetValue(NULL,"-viewer_binary_mpiio","");CHKERRQ(ierr);
}
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"temp",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HDF5)
} else if (ishdf5) {
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"temp",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
#endif
} else SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Invalid Viewer : Run with -binary or -hdf5 option\n");
ierr = VecView(global1,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Global vector written to temp file is \n");CHKERRQ(ierr);
ierr = VecView(global1,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (flg2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,stencil_type,M,N,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,&da2);CHKERRQ(ierr);
} else if (flg3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD,bx,by,bz,stencil_type,M,N,P,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,dof,stencil_width,0,0,0,&da2);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(PETSC_COMM_WORLD,bx,M,dof,stencil_width,NULL,&da2);CHKERRQ(ierr);
}
if (isbinary) {
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"temp",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HDF5)
} else if (ishdf5) {
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"temp",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
#endif
} else SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Invalid Viewer : Run with -binary or -hdf5 option\n");
ierr = DMCreateGlobalVector(da2,&global2);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)global2,"Test_Vec");CHKERRQ(ierr);
ierr = VecLoad(global2,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Global vector read from temp file is \n");CHKERRQ(ierr);
ierr = VecView(global2,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecAXPY(global2,mone,global1);CHKERRQ(ierr);
ierr = VecNorm(global2,NORM_MAX,&norm);CHKERRQ(ierr);
if (norm != 0.0) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"ex23: Norm of difference %g should be zero\n",(double)norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Number of processors %d\n",size);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," M,N,P,dof %D %D %D %D\n",M,N,P,dof);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," stencil_width %D stencil_type %d periodic %d\n",stencil_width,(int)stencil_type,(int)pt);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," dimension %d\n",1 + (int) flg2 + (int) flg3);CHKERRQ(ierr);
}
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = DMDestroy(&da2);CHKERRQ(ierr);
ierr = VecDestroy(&global1);CHKERRQ(ierr);
ierr = VecDestroy(&global2);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
krylov_petsc_info_t
krylov_petsc_solve
(
p4est_t* p4est,
problem_data_t* vecs,
weakeqn_ptrs_t* fcns,
p4est_ghost_t** ghost,
element_data_t** ghost_data,
dgmath_jit_dbase_t* dgmath_jit_dbase,
krylov_petsc_params_t* krylov_params
)
{
krylov_petsc_info_t info;
KSP ksp;
Vec x,b;
PC pc;
/* double* u_temp; */
/* double* rhs_temp; */
KSPCreate(PETSC_COMM_WORLD,&ksp);
VecCreate(PETSC_COMM_WORLD,&x);//CHKERRQ(ierr);
VecSetSizes(x, vecs->local_nodes, PETSC_DECIDE);//CHKERRQ(ierr);
VecSetFromOptions(x);//CHKERRQ(ierr);
VecDuplicate(x,&b);//CHKERRQ(ierr);
/* VecGetArray(x,&u_temp); */
/* VecGetArray(b,&rhs_temp); */
krylov_pc_ctx_t kct;
kct.p4est = p4est;
kct.vecs = vecs;
kct.fcns = fcns;
kct.ghost = ghost;
kct.ghost_data = ghost_data;
kct.dgmath_jit_dbase = dgmath_jit_dbase;
kct.pc_data = krylov_params->pc_data;
if (krylov_params->ksp_monitor)
PetscOptionsSetValue(NULL,"-ksp_monitor","");
if (krylov_params->ksp_monitor)
PetscOptionsSetValue(NULL,"-ksp_view","");
/* KSPMonitorSet(ksp, KSPMonitorDefault, NULL, NULL); */
/* PetscOptionsSetValue(NULL,"-ksp_converged_reason",""); */
PetscOptionsSetValue(NULL,"-ksp_atol","1e-20");
/* PetscOptionsSetValue(NULL,"-with-debugging","1"); */
PetscOptionsSetValue(NULL,"-ksp_rtol","1e-20");
PetscOptionsSetValue(NULL,"-ksp_max_it","1000000");
KSPGetPC(ksp,&pc);
krylov_pc_t* kp = NULL;
if (krylov_params != NULL && krylov_params->user_defined_pc) {
PCSetType(pc,PCSHELL);//CHKERRQ(ierr);
kp = krylov_params->pc_create(&kct);
PCShellSetApply(pc, krylov_petsc_pc_apply);//CHKERRQ(ierr);
PCShellSetSetUp(pc, krylov_petsc_pc_setup);
PCShellSetContext(pc, kp);//CHKERRQ(ierr);
}
else {
PCSetType(pc,PCNONE);//CHKERRQ(ierr);
}
KSPSetType(ksp, krylov_params->krylov_type);
KSPSetFromOptions(ksp);
/* Create matrix-free shell for Aij */
Mat A;
MatCreateShell
(
PETSC_COMM_WORLD,
vecs->local_nodes,
vecs->local_nodes,
PETSC_DETERMINE,
PETSC_DETERMINE,
(void*)&kct,
&A
);
MatShellSetOperation(A,MATOP_MULT,(void(*)())krylov_petsc_apply_aij);
/* Set Amat and Pmat, where Pmat is the matrix the Preconditioner needs */
KSPSetOperators(ksp,A,A);
/* linalg_copy_1st_to_2nd(vecs->u, u_temp, vecs->local_nodes); */
/* linalg_copy_1st_to_2nd(vecs->rhs, rhs_temp, vecs->local_nodes); */
VecPlaceArray(b, vecs->rhs);
VecPlaceArray(x, vecs->u);
KSPSolve(ksp,b,x);
if (krylov_params != NULL && krylov_params->user_defined_pc) {
krylov_params->pc_destroy(kp);
}
KSPGetIterationNumber(ksp, &(info.iterations));
KSPGetResidualNorm(ksp, &(info.residual_norm));
/* linalg_copy_1st_to_2nd(u_temp, vecs->u, vecs->local_nodes); */
/* VecRestoreArray(x,&u_temp); */
/* VecRestoreArray(b,&rhs_temp); */
VecResetArray(b);
VecResetArray(x);
VecDestroy(&x);
VecDestroy(&b);
KSPDestroy(&ksp);
return info;
}
int main(int argc,char **args)
{
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInt i,m = 10,low,high,ldim,iglobal;
PetscScalar v;
Vec u;
PetscViewer viewer;
#if defined(PETSC_USE_LOG)
PetscLogEvent VECTOR_GENERATE,VECTOR_READ;
#endif
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
/* PART 1: Generate vector, then write it in binary format */
ierr = PetscLogEventRegister("Generate Vector",VEC_CLASSID,&VECTOR_GENERATE);CHKERRQ(ierr);
ierr = PetscLogEventBegin(VECTOR_GENERATE,0,0,0,0);CHKERRQ(ierr);
/* Generate vector */
ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr);
ierr = VecSetSizes(u,PETSC_DECIDE,m);CHKERRQ(ierr);
ierr = VecSetFromOptions(u);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(u,&low,&high);CHKERRQ(ierr);
ierr = VecGetLocalSize(u,&ldim);CHKERRQ(ierr);
for (i=0; i<ldim; i++) {
iglobal = i + low;
v = (PetscScalar)(i + 100*rank);
ierr = VecSetValues(u,1,&iglobal,&v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(u);CHKERRQ(ierr);
ierr = VecAssemblyEnd(u);CHKERRQ(ierr);
ierr = VecView(u,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"writing vector in binary to vector.dat ...\n");CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"vector.dat",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
ierr = VecView(u,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
/* ierr = PetscOptionsClear();CHKERRQ(ierr);*/
ierr = PetscOptionsSetValue("-viewer_binary_mpiio","");CHKERRQ(ierr);
ierr = PetscLogEventEnd(VECTOR_GENERATE,0,0,0,0);CHKERRQ(ierr);
/* PART 2: Read in vector in binary format */
/* Read new vector in binary format */
ierr = PetscLogEventRegister("Read Vector",VEC_CLASSID,&VECTOR_READ);CHKERRQ(ierr);
ierr = PetscLogEventBegin(VECTOR_READ,0,0,0,0);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"reading vector in binary from vector.dat ...\n");CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"vector.dat",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr);
ierr = VecLoad(u,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscLogEventEnd(VECTOR_READ,0,0,0,0);CHKERRQ(ierr);
ierr = VecView(u,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Free data structures */
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscInt i,m = 20,low,high,ldim,iglobal,lsize;
PetscScalar v;
Vec u;
PetscViewer viewer;
PetscBool vstage2,vstage3,mpiio_use,isbinary,ishdf5;
#if defined(PETSC_USE_LOG)
PetscLogEvent VECTOR_GENERATE,VECTOR_READ;
#endif
PetscInitialize(&argc,&args,(char*)0,help);
isbinary = ishdf5 = PETSC_FALSE;
mpiio_use = vstage2 = vstage3 = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-binary",&isbinary,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-hdf5",&ishdf5,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-mpiio",&mpiio_use,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-sizes_set",&vstage2,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-type_set",&vstage3,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr);
/* PART 1: Generate vector, then write it in the given data format */
ierr = PetscLogEventRegister("Generate Vector",VEC_CLASSID,&VECTOR_GENERATE);CHKERRQ(ierr);
ierr = PetscLogEventBegin(VECTOR_GENERATE,0,0,0,0);CHKERRQ(ierr);
/* Generate vector */
ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)u, "Test_Vec");CHKERRQ(ierr);
ierr = VecSetSizes(u,PETSC_DECIDE,m);CHKERRQ(ierr);
ierr = VecSetFromOptions(u);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(u,&low,&high);CHKERRQ(ierr);
ierr = VecGetLocalSize(u,&ldim);CHKERRQ(ierr);
for (i=0; i<ldim; i++) {
iglobal = i + low;
v = (PetscScalar)(i + 100*rank);
ierr = VecSetValues(u,1,&iglobal,&v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(u);CHKERRQ(ierr);
ierr = VecAssemblyEnd(u);CHKERRQ(ierr);
ierr = VecView(u,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (isbinary) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"writing vector in binary to vector.dat ...\n");CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"vector.dat",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HDF5)
} else if (ishdf5) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"writing vector in hdf5 to vector.dat ...\n");CHKERRQ(ierr);
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"vector.dat",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
#endif
} else SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"No data format specified, run with either -binary or -hdf5 option\n");
ierr = VecView(u,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = PetscLogEventEnd(VECTOR_GENERATE,0,0,0,0);CHKERRQ(ierr);
/* PART 2: Read in vector in binary format */
/* Read new vector in binary format */
ierr = PetscLogEventRegister("Read Vector",VEC_CLASSID,&VECTOR_READ);CHKERRQ(ierr);
ierr = PetscLogEventBegin(VECTOR_READ,0,0,0,0);CHKERRQ(ierr);
if (mpiio_use) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Using MPI IO for reading the vector\n");CHKERRQ(ierr);
ierr = PetscOptionsSetValue(NULL,"-viewer_binary_mpiio","");CHKERRQ(ierr);
}
if (isbinary) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"reading vector in binary from vector.dat ...\n");CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"vector.dat",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = PetscViewerBinarySetFlowControl(viewer,2);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HDF5)
} else if (ishdf5) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"reading vector in hdf5 from vector.dat ...\n");CHKERRQ(ierr);
ierr = PetscViewerHDF5Open(PETSC_COMM_WORLD,"vector.dat",FILE_MODE_READ,&viewer);CHKERRQ(ierr);
#endif
}
ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) u,"Test_Vec");
if (vstage2) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Setting vector sizes...\n");CHKERRQ(ierr);
if (size > 1) {
if (!rank) {
lsize = m/size + size;
ierr = VecSetSizes(u,lsize,m);CHKERRQ(ierr);
} else if (rank == size-1) {
lsize = m/size - size;
ierr = VecSetSizes(u,lsize,m);CHKERRQ(ierr);
} else {
lsize = m/size;
ierr = VecSetSizes(u,lsize,m);CHKERRQ(ierr);
}
} else {
ierr = VecSetSizes(u,m,m);CHKERRQ(ierr);
}
}
if (vstage3) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Setting vector type...\n");CHKERRQ(ierr);
ierr = VecSetType(u, VECMPI);CHKERRQ(ierr);
}
ierr = VecLoad(u,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscLogEventEnd(VECTOR_READ,0,0,0,0);CHKERRQ(ierr);
ierr = VecView(u,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Free data structures */
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void
Split::setup(const std::string& prefix)
{
PetscErrorCode ierr;
std::string dmprefix = prefix + "dm_moose_", opt, val;
// var options
if (_vars.size()) {
opt = dmprefix + "vars";
val="";
for (unsigned int j = 0; j < _vars.size(); ++j)
val += (j ? "," : "") + _vars[j];
ierr = PetscOptionsSetValue(opt.c_str(), val.c_str());
CHKERRABORT(_communicator.get(), ierr);
}
// block options
if (_blocks.size()) {
opt = dmprefix + "blocks";
val = "";
for (unsigned int j = 0; j < _blocks.size(); ++j)
val += (j ? "," : "") + _blocks[j];
ierr = PetscOptionsSetValue(opt.c_str(), val.c_str());
CHKERRABORT(_communicator.get(), ierr);
}
// side options
if (_sides.size()) {
opt = dmprefix + "sides";
val = "";
for (unsigned int j = 0; j < _sides.size(); ++j)
val += (j ? "," : "") + _sides[j];
ierr = PetscOptionsSetValue(opt.c_str(), val.c_str());
CHKERRABORT(_communicator.get(), ierr);
}
// unside options
if (_unsides.size()) {
opt = dmprefix + "unsides";
val = "";
for (unsigned int j = 0; j < _unsides.size(); ++j)
val += (j ? "," : "") + _unsides[j];
ierr = PetscOptionsSetValue(opt.c_str(), val.c_str());
CHKERRABORT(_communicator.get(), ierr);
}
if (_splitting.size()) {
// If this split has subsplits, it is presumed that the pc_type used to solve this split's subsystem is fieldsplit
// with the following parameters (unless overridden by the user-specified petsc_options below).
opt = prefix + "pc_type";
val = "fieldsplit";
ierr = PetscOptionsSetValue(opt.c_str(), val.c_str());
CHKERRABORT(_communicator.get(), ierr);
// set Splitting Type
const char * petsc_splitting_type[] = {
"additive",
"multiplicative",
"symmetric_multiplicative",
"schur"
};
opt = prefix + "pc_fieldsplit_type";
ierr = PetscOptionsSetValue(opt.c_str(), petsc_splitting_type[_splitting_type]);
CHKERRABORT(_communicator.get(), ierr);
if (_splitting_type == SplittingTypeSchur)
{
// set Schur Type
const char * petsc_schur_type[] = {
"diag",
"upper",
"lower",
"full"
};
opt = prefix + "pc_fieldsplit_schur_fact_type";
ierr = PetscOptionsSetValue(opt.c_str(), petsc_schur_type[_splitting_type]);
CHKERRABORT(_communicator.get(), ierr);
// set Schur Preconditioner
const char * petsc_schur_pre[] = {
"self",
"selfp",
#if PETSC_VERSION_LESS_THAN(3,4,0)
"diag"
#else
"a11"
#endif
};
opt = prefix + "pc_fieldsplit_schur_precondition";
ierr = PetscOptionsSetValue(opt.c_str(), petsc_schur_pre[_schur_pre]);
CHKERRABORT(_communicator.get(), ierr);
// set Schur AInv
const char * petsc_schur_ainv[] = {
"diag",
"lump"
};
opt = prefix + "mat_schur_complement_ainv_type";
ierr = PetscOptionsSetValue(opt.c_str(), petsc_schur_ainv[_schur_ainv]);
CHKERRABORT(_communicator.get(), ierr);
}
// FIXME: How would we support the user-provided Pmat?
// The DM associated with this split defines the subsplits' geometry.
opt = dmprefix + "nfieldsplits";
std::ostringstream sval;
sval << _splitting.size();
val = sval.str();
ierr = PetscOptionsSetValue(opt.c_str(), val.c_str());
CHKERRABORT(_communicator.get(), ierr);
opt = dmprefix + "fieldsplit_names";
val = "";
for (unsigned int i = 0; i < _splitting.size(); ++i)
val += (i ? "," : "") + _splitting[i];
ierr = PetscOptionsSetValue(opt.c_str(), val.c_str());
CHKERRABORT(_communicator.get(), ierr);
// Finally, recursively configure the splits contained within this split.
for (unsigned int i = 0; i < _splitting.size(); ++i)
{
Split* split = _fe_problem.getNonlinearSystem().getSplit(_splitting[i]);
std::string sprefix = prefix + "fieldsplit_" + _splitting[i] + "_";
split->setup(sprefix);
}
}
// Now we set the user-specified petsc options for this split, possibly overriding the above settings.
for (unsigned j = 0; j < _petsc_options.size(); ++j)
{
// Need to prepend the prefix and strip off the leading '-' on the option name.
const std::string & op = _petsc_options[j];
if (op[0] != '-')
mooseError("Invalid petsc option name " << op << " for Split " << _name);
std::string opt = prefix + op.substr(1);
ierr = PetscOptionsSetValue(opt.c_str(), PETSC_NULL);
CHKERRABORT(_communicator.get(), ierr);
}
for (unsigned j = 0; j < _petsc_options_iname.size(); ++j)
{
// Need to prepend the prefix and strip off the leading '-' on the option name.
const std::string & op = _petsc_options_iname[j];
if (op[0] != '-')
mooseError("Invalid petsc option name " << op << " for Split " << _name);
std::string opt = prefix + op.substr(1);
ierr = PetscOptionsSetValue(opt.c_str(), _petsc_options_value[j].c_str());
CHKERRABORT(_communicator.get(), ierr);
}
}
int main(int argc,char **argv)
{
KSP ksp;
PC pc;
Mat A,M;
Vec X,B,D;
MPI_Comm comm;
PetscScalar v;
KSPConvergedReason reason;
PetscInt i,j,its;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscInitialize(&argc,&argv,0,help);CHKERRQ(ierr);
ierr = PetscOptionsSetValue("-options_left",PETSC_NULL);CHKERRQ(ierr);
comm = MPI_COMM_SELF;
/*
* Construct the Kershaw matrix
* and a suitable rhs / initial guess
*/
ierr = MatCreateSeqAIJ(comm,4,4,4,0,&A);CHKERRQ(ierr);
ierr = VecCreateSeq(comm,4,&B);CHKERRQ(ierr);
ierr = VecDuplicate(B,&X);CHKERRQ(ierr);
for (i=0; i<4; i++) {
v=3;
ierr = MatSetValues(A,1,&i,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
v=1;
ierr = VecSetValues(B,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecSetValues(X,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
}
i=0; v=0;
ierr = VecSetValues(X,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
for (i=0; i<3; i++) {
v=-2; j=i+1;
ierr = MatSetValues(A,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(A,1,&j,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
}
i=0; j=3; v=2;
ierr = MatSetValues(A,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(A,1,&j,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecAssemblyBegin(B);CHKERRQ(ierr);
ierr = VecAssemblyEnd(B);CHKERRQ(ierr);
printf("\nThe Kershaw matrix:\n\n"); MatView(A,0);
/*
* A Conjugate Gradient method
* with ILU(0) preconditioning
*/
ierr = KSPCreate(comm,&ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPCG);CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp,PETSC_TRUE);CHKERRQ(ierr);
/*
* ILU preconditioner;
* The iterative method will break down unless you comment in the SetShift
* line below, or use the -pc_factor_shift_positive_definite option.
* Run the code twice: once as given to see the negative pivot and the
* divergence behaviour, then comment in the Shift line, or add the
* command line option, and see that the pivots are all positive and
* the method converges.
*/
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCICC);CHKERRQ(ierr);
/* ierr = PCFactorSetShiftType(prec,MAT_SHIFT_POSITIVE_DEFINITE);CHKERRQ(ierr); */
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
/*
* Now that the factorisation is done, show the pivots;
* note that the last one is negative. This in itself is not an error,
* but it will make the iterative method diverge.
*/
ierr = PCFactorGetMatrix(pc,&M);CHKERRQ(ierr);
ierr = VecDuplicate(B,&D);CHKERRQ(ierr);
ierr = MatGetDiagonal(M,D);CHKERRQ(ierr);
printf("\nPivots:\n\n"); VecView(D,0);
/*
* Solve the system;
* without the shift this will diverge with
* an indefinite preconditioner
*/
ierr = KSPSolve(ksp,B,X);CHKERRQ(ierr);
ierr = KSPGetConvergedReason(ksp,&reason);CHKERRQ(ierr);
if (reason==KSP_DIVERGED_INDEFINITE_PC) {
printf("\nDivergence because of indefinite preconditioner;\n");
printf("Run the executable again but with -pc_factor_shift_positive_definite option.\n");
} else if (reason<0) {
printf("\nOther kind of divergence: this should not happen.\n");
} else {
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
printf("\nConvergence in %d iterations.\n",(int)its);
}
printf("\n");
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = VecDestroy(&B);CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&D);CHKERRQ(ierr);
PetscFinalize();
PetscFunctionReturn(0);
}
void TestChebyshevAdaptiveVsNoAdaptive() throw (Exception)
{
unsigned num_nodes = 1331;
DistributedVectorFactory factory(num_nodes);
Vec parallel_layout = factory.CreateVec(2);
// Solving with zero guess for coverage
Vec zero_guess = factory.CreateVec(2);
double zero = 0.0;
#if (PETSC_VERSION_MAJOR == 2 && PETSC_VERSION_MINOR == 2)
VecSet(&zero, zero_guess);
#else
VecSet(zero_guess, zero);
#endif
Mat system_matrix;
// Note that this test deadlocks if the file's not on the disk
PetscTools::ReadPetscObject(system_matrix, "linalg/test/data/matrices/cube_6000elems_half_activated.mat", parallel_layout);
Vec system_rhs;
// Note that this test deadlocks if the file's not on the disk
PetscTools::ReadPetscObject(system_rhs, "linalg/test/data/matrices/cube_6000elems_half_activated.vec", parallel_layout);
// Make sure we are not inheriting a non-default number of iterations from previous test
std::stringstream num_it_str;
num_it_str << 1000;
PetscOptionsSetValue("-ksp_max_it", num_it_str.str().c_str());
try
{
LinearSystem ls = LinearSystem(system_rhs, system_matrix);
ls.SetMatrixIsSymmetric();
// Solve to relative convergence for coverage
ls.SetRelativeTolerance(1e-6);
ls.SetPcType("jacobi");
ls.SetKspType("chebychev");
ls.SetUseFixedNumberIterations(true, 64);
// Solving with zero guess for coverage.
Vec solution = ls.Solve(zero_guess);
unsigned chebyshev_adaptive_its = ls.GetNumIterations();
TS_ASSERT_EQUALS(chebyshev_adaptive_its, 40u);
TS_ASSERT_DELTA(ls.mEigMin, 0.0124, 1e-4);
TS_ASSERT_DELTA(ls.mEigMax, 1.8810, 1e-4);
PetscTools::Destroy(solution);
}
catch (Exception& e)
{
if (e.GetShortMessage() == "Chebyshev with fixed number of iterations is known to be broken in PETSc <= 2.3.2")
{
WARNING(e.GetShortMessage());
}
else
{
TS_FAIL(e.GetShortMessage());
}
}
// Make sure we are not inheriting a non-default number of iterations from previous test
PetscOptionsSetValue("-ksp_max_it", num_it_str.str().c_str());
{
LinearSystem ls = LinearSystem(system_rhs, system_matrix);
ls.SetMatrixIsSymmetric();
ls.SetRelativeTolerance(1e-6);
ls.SetPcType("jacobi");
ls.SetKspType("chebychev");
Vec solution = ls.Solve(zero_guess);
unsigned chebyshev_no_adaptive_its = ls.GetNumIterations();
TS_ASSERT_LESS_THAN(chebyshev_no_adaptive_its, 100u); // Normally 88, but 99 on maverick & natty
TS_ASSERT_DELTA(ls.mEigMin, 0.0124, 1e-4);
TS_ASSERT_DELTA(ls.mEigMax, 1.8841, 1e-4);
PetscTools::Destroy(solution);
}
// Make sure we are not inheriting a non-default number of iterations from previous test
PetscOptionsSetValue("-ksp_max_it", num_it_str.str().c_str());
{
LinearSystem ls = LinearSystem(system_rhs, system_matrix);
ls.SetMatrixIsSymmetric();
ls.SetRelativeTolerance(1e-6);
ls.SetPcType("jacobi");
ls.SetKspType("cg");
Vec solution = ls.Solve(zero_guess);
unsigned cg_its = ls.GetNumIterations();
TS_ASSERT_EQUALS(cg_its, 40u);
TS_ASSERT_EQUALS(ls.mEigMin, DBL_MAX);
TS_ASSERT_EQUALS(ls.mEigMax, DBL_MIN);
PetscTools::Destroy(solution);
}
PetscTools::Destroy(system_matrix);
PetscTools::Destroy(system_rhs);
PetscTools::Destroy(parallel_layout);
PetscTools::Destroy(zero_guess);
}
/**
* Routine for solving a linear equation with PETSc.
*
* Note that this routine calls MPI teardown routines, so it should
* not be called in the main process unless you want to break MPI for
* the remainder of the process lifetime.
*
*/
PetscErrorCode _solve_routine(JNIEnv *env, jint n, jintArray *index,
jobjectArray *diagonals, jobjectArray *options,
double *solution, jdoubleArray *rhs) {
PetscErrorCode ierr;
KSP ksp;
PC pc;
Mat A;
Vec b;
Vec x;
PetscInitialize(0, NULL, (char*) NULL, NULL);
// Set up matrix A in Ax = b
int num_diags = (*env) -> GetArrayLength(env, *index);
ierr = MatCreateSeqAIJ(PETSC_COMM_WORLD, n, n, num_diags, NULL,
&A); CHKERRQ(ierr);
ierr = MatSetUp(A); CHKERRQ(ierr);
ierr = _fill_matrix(env, &A, index, diagonals);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
// Set up vectors b and x in Ax = b
ierr = VecCreate(PETSC_COMM_WORLD, &b); CHKERRQ(ierr);
ierr = VecSetSizes(b, PETSC_DECIDE, n); CHKERRQ(ierr);
ierr = VecSetType(b, VECSEQ);
ierr = VecDuplicate(b, &x);
ierr = _fill_vector(env, &b, rhs); CHKERRQ(ierr);
ierr = VecAssemblyBegin(b); CHKERRQ(ierr);
ierr = VecAssemblyEnd(b); CHKERRQ(ierr);
// Set up solver
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp); CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr);
// Solver options
ierr = KSPGetPC(ksp, &pc); CHKERRQ(ierr);
for(int i = 0; i < (*env)->GetArrayLength(env, *options); i++) {
jobject option = (*env)->GetObjectArrayElement(env, *options, i);
jstring key = (*env)->GetObjectArrayElement(env, option, 0);
jstring value = (*env)->GetObjectArrayElement(env, option, 1);
const jchar *keyChars = (*env)->GetStringChars(env, key, NULL);
const jchar *valChars = (*env)->GetStringChars(env, value, NULL);
PetscOptionsSetValue((char*) keyChars, (char*) valChars);
(*env)->ReleaseStringChars(env, key, keyChars);
(*env)->ReleaseStringChars(env, value, valChars);
}
ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr);
ierr = KSPSetUp(ksp); CHKERRQ(ierr);
// Solve
ierr = KSPSolve(ksp, b, x); CHKERRQ(ierr);
// Copy solution into JNI allocated memory
ierr = _pvec_to_array(&x, solution); CHKERRQ(ierr);
// Clean up and return
ierr = MatDestroy(&A); CHKERRQ(ierr);
ierr = VecDestroy(&b); CHKERRQ(ierr);
ierr = VecDestroy(&x); CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);
PetscFinalize();
return ierr;
}
void TestBidomain3d() throw (Exception)
{
HeartConfig::Instance()->SetIntracellularConductivities(Create_c_vector(1.75, 1.75, 1.75));
HeartConfig::Instance()->SetExtracellularConductivities(Create_c_vector(7.0, 7.0, 7.0));
HeartConfig::Instance()->SetSimulationDuration(150.0); //ms
//Note that we can only call the old permute nodes funcutionality on the sequential mesh object
//HeartConfig::Instance()->SetMeshFileName("mesh/test/data/3D_0_to_.5mm_1889_elements_irregular");
BidomainFaceStimulusCellFactory bidomain_cell_factory;
BidomainProblem<3> bidomain_problem( &bidomain_cell_factory );
TetrahedralMesh<3,3> mesh;
TrianglesMeshReader<3,3> mesh_reader("mesh/test/data/3D_0_to_.5mm_1889_elements_irregular");
mesh.ConstructFromMeshReader(mesh_reader);
bidomain_problem.SetMesh(&mesh);
bidomain_problem.PrintOutput(false);
HeartConfig::Instance()->SetKSPSolver("symmlq");
HeartConfig::Instance()->SetKSPPreconditioner("bjacobi");
PetscOptionsSetValue("-log_summary", "");
bidomain_problem.Initialise();
//Mesh isn't actually loaded until initialise method is called
RandomNumberGenerator::Instance();
bidomain_problem.rGetMesh().PermuteNodes();
RandomNumberGenerator::Destroy();
bidomain_problem.Solve();
Vec voltage=bidomain_problem.GetSolution();
ReplicatableVector voltage_replicated;
voltage_replicated.ReplicatePetscVector(voltage);
/*
* Test the top right node against the right one in the 1D case,
* comparing voltage, and then test all the nodes on the right hand
* face of the cube against the top right one, comparing voltage.
*/
bool need_initialisation = true;
double probe_voltage=-9999.;
need_initialisation = true;
// Test the RHF of the mesh
for (unsigned i = 0; i < bidomain_problem.rGetMesh().GetNumNodes(); i++)
{
if (bidomain_problem.rGetMesh().GetNode(i)->GetPoint()[0] == 0.05)
{
// x = 0 is where the stimulus has been applied
// x = 0.05cm is the other end of the mesh and where we want to
// to test the value of the nodes
if (need_initialisation)
{
probe_voltage = voltage_replicated[2*i];
need_initialisation = false;
}
else
{
// the voltage at the end face varies a little because
// of drift due to the orientation of the tets in the mesh,
// hence the tolerance of 0.02
TS_ASSERT_DELTA(voltage_replicated[2*i], probe_voltage, 0.02);
}
// Check against hard coded value
// For 50 ms test TS_ASSERT_DELTA(voltage_replicated[2*i], 7.3, 0.2);
// For 150 ms test
TS_ASSERT_DELTA(voltage_replicated[2*i], -1.735, 0.1);
}
}
}
PetscErrorCode efs_setup(efs *slv, int offset[], int stride[])
{
PetscErrorCode ierr;
PetscInt xs, ys, zs, xm, ym, zm;
PCType pc_type;
if (efs_log(slv, EFS_LOG_STATUS)) {
ierr = ef_io_print(slv->comm, "Setting up electric field solver");CHKERRQ(ierr);
}
slv->ts = 0;
if (efs_log(slv, EFS_LOG_RESIDUAL)) {
ierr = PetscOptionsSetValue("-ksp_monitor_short", NULL);CHKERRQ(ierr);
}
ierr = DMDASetFieldName(slv->dm, 0,"potential");CHKERRQ(ierr);
if (slv->grid.nd == 2) {
ierr = DMDAGetCorners(slv->dm, &xs, &ys, 0, &xm, &ym, 0);CHKERRQ(ierr);
slv->dmap = ef_dmap_create_2d(xs - offset[0], ys - offset[1], xm, ym, stride);
} else if (slv->grid.nd == 3) {
ierr = DMDAGetCorners(slv->dm, &xs, &ys, &zs, &xm, &ym, &zm);CHKERRQ(ierr);
slv->dmap = ef_dmap_create_3d(xs - offset[0], ys - offset[1], zs - offset[2], xm, ym, zm, stride);
} else {
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_SUP, "Unsupported dimmension: %d", slv->grid.nd);
}
ierr = ef_callback_create(&slv->callback);CHKERRQ(ierr);
ierr = KSPCreate(slv->comm, &slv->ksp);CHKERRQ(ierr);
if (efs_log(slv, EFS_LOG_EIGS)) {
ierr = KSPSetComputeEigenvalues(slv->ksp, PETSC_TRUE);CHKERRQ(ierr);
}
ierr = KSPSetDM(slv->ksp, slv->dm);CHKERRQ(ierr);
ierr = KSPGetPC(slv->ksp, &slv->pc);CHKERRQ(ierr);
ierr = PCSetType(slv->pc, PCMG);CHKERRQ(ierr);
if (slv->options.galerkin) {
ierr = PCMGSetGalerkin(slv->pc, PETSC_TRUE);CHKERRQ(ierr);
} else {
ierr = PCMGSetGalerkin(slv->pc, PETSC_FALSE);CHKERRQ(ierr);
}
ierr = KSPSetComputeOperators(slv->ksp, slv->callback->matrix, slv);CHKERRQ(ierr);
ierr = KSPSetComputeRHS(slv->ksp, slv->callback->rhs, slv);CHKERRQ(ierr);
ierr = KSPSetComputeInitialGuess(slv->ksp, slv->callback->guess, slv);CHKERRQ(ierr);
ierr = KSPSetFromOptions(slv->ksp);CHKERRQ(ierr);
ierr = PCGetType(slv->pc, &pc_type);CHKERRQ(ierr);
ierr = PCMGGetLevels(slv->pc, &slv->options.levels);CHKERRQ(ierr);
if (slv->options.levels < 1) slv->options.levels++;
ierr = PCMGGetGalerkin(slv->pc, &slv->options.galerkin);CHKERRQ(ierr);
if (strcmp(pc_type, PCGAMG) == 0
|| strcmp(pc_type, PCHYPRE) == 0) slv->options.galerkin = 1;
if (!slv->options.galerkin) {
slv->levels = (ef_level*) malloc(slv->options.levels*sizeof(ef_level));
// setup callback for transforming rhs on coarse levels
} else {
slv->levels = (ef_level*) malloc(sizeof(ef_level));
}
ierr = ef_fd_create(&slv->fd, EF_FD_STANDARD_O2);CHKERRQ(ierr);
ierr = ef_operator_create(&slv->op, slv->levels, slv->fd, slv->grid.nd);CHKERRQ(ierr);
ierr = ef_boundary_create(&slv->boundary, slv->levels, slv->options.levels,
slv->dmap, &slv->state, slv->fd);CHKERRQ(ierr);
slv->op->axisymmetric = slv->options.axisymmetric;
slv->boundary->axisymmetric = slv->options.axisymmetric;
ierr = DMSetMatType(slv->dm, MATAIJ);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].eps);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].g);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].ag);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].gcomp);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].scale);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].nscale);CHKERRQ(ierr);
ierr = VecSet(slv->levels[0].g, 0);CHKERRQ(ierr);
ierr = VecSet(slv->levels[0].gcomp, 1);CHKERRQ(ierr);
ierr = VecSet(slv->levels[0].scale, 1);CHKERRQ(ierr);
ierr = VecSet(slv->levels[0].nscale, 1);CHKERRQ(ierr);
ierr = DMCoarsenHookAdd(slv->dm, slv->callback->coarsen, slv->callback->restrct, slv);CHKERRQ(ierr);
return 0;
}
/* This function returns the suitable solver for pressure. linear system */
KSP Solver_get_pressure_solver(Mat lhs, Parameters *params) {
KSP solver;
PC pc;
double rtol;
int ierr;
PetscLogDouble T1, T2;
rtol = params->resmax;
ierr = KSPCreate(PETSC_COMM_WORLD, &solver); PETScErrAct(ierr);
short int hasnullspace = NO;
if (hasnullspace) {
MatNullSpace nullsp;
MatNullSpaceCreate(PETSC_COMM_WORLD, PETSC_TRUE, 0, PETSC_NULL, &nullsp);
KSPSetNullSpace(solver, nullsp);
//MatNullSpaceDestroy(nullsp);
}
ierr = KSPSetOperators(solver, lhs, lhs, SAME_PRECONDITIONER); PETScErrAct(ierr);
ierr = KSPSetType(solver, KSPGMRES); PETScErrAct(ierr);
ierr = KSPGMRESSetRestart(solver, 20); PETScErrAct(ierr);
ierr = KSPSetTolerances(solver, rtol, PETSC_DEFAULT, PETSC_DEFAULT, 300); PETScErrAct(ierr);
ierr = KSPGetPC(solver, &pc); PETScErrAct(ierr);
//PCSetType(pc, PCNONE);
//PCSetType(pc, PCASM);
PCSetType(pc, PCHYPRE);
PCHYPRESetType(pc,"boomeramg");
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_max_levels", "25"); PETScErrAct(ierr);
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_strong_threshold", "0.0"); PETScErrAct(ierr);
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_relax_type_all", "SOR/Jacobi"); PETScErrAct(ierr);
//ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_cycle_type", ""); PETScErrAct(ierr);
//ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_cycle_type", "V"); PETScErrAct(ierr);
//ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_coarsen_type", "PMIS"); PETScErrAct(ierr);
//ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_truncfactor", "0.9"); PETScErrAct(ierr);
/*******************************************************************************************************/
/*******************************************************************************************************/
/*******************************************************************************************************/
/* Hypre-Petsc Interface.
The most important parameters to be set are
1- Strong Threshold
2- Truncation Factor
3- Coarsennig Type
*/
/* Between 0 to 1 */
/* "0 "gives better convergence rate (in 3D). */
/* Suggested values (By Hypre manual): 0.25 for 2D, 0.5 for 3D
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_strong_threshold", "0.0"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Available Options:
"CLJP","Ruge-Stueben","modifiedRuge-Stueben","Falgout", "PMIS", "HMIS"
Falgout is usually the best.
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_coarsen_type", "Falgout"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Availble options:
"local", "global"
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_measure_type", "local"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Availble options:
Jacobi,sequential-Gauss-Seidel,
SOR/Jacobi,backward-SOR/Jacobi,symmetric-SOR/Jacobi,Gaussian-elimination
Important: If you are using a symmetric KSP solver (like CG), you should use a symmetric smoother
here.
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_relax_type_all", "symmetric-SOR/Jacobi"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Available options:
"V", "W"
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_cycle_type", "V"); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Availble options:
"classical", "", "", "direct", "multipass", "multipass-wts", "ext+i", "ext+i-cc", "standard", "standard-wts", "", "", "FF", "FF1"
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_interp_type", ""); PETScErrAct(ierr);
*/
/*******************************************************************************************************/
/* Available options:
Greater than zero.
Use zero for the best convergence. However, if you have memory problems, use greate than zero to save some
memory.
ierr = PetscOptionsSetValue("-pc_hypre_boomeramg_truncfactor", "0.0"); PETScErrAct(ierr);
*/
/*
Preconditioner Generation Options
PCSetType(pc,PCHYPRE) or -pc_type hypre
-pc_hypre_boomeramg_max_levels nmax
-pc_hypre_boomeramg_truncfactor
-pc_hypre_boomeramg_strong_threshold
-pc_hypre_boomeramg_max_row_sum
-pc_hypre_boomeramg_no_CF
-pc_hypre_boomeramg_coarsen_type CLJP,Ruge-Stueben,modifiedRuge-Stueben,
-pc_hypre_boomeramg_measure_type local,global
Preconditioner Iteration Options
-pc_hypre_boomeramg_relax_type_all Jacobi,sequential-Gauss-Seidel,
SOR/Jacobi,backward-SOR/Jacobi,symmetric-SOR/Jacobi,Gaussian-eliminat
-pc_hypre_boomeramg_relax_type_fine
-pc_hypre_boomeramg_relax_type_down
-pc_hypre_boomeramg_relax_type_up
-pc_hypre_boomeramg_relax_weight_all r
-pc_hypre_boomeramg_outer_relax_weight_all r
-pc_hypre_boomeramg_grid_sweeps_down n
-pc_hypre_boomeramg_grid_sweeps_up n
-pc_hypre_boomeramg_grid_sweeps_coarse n
-pc_hypre_boomeramg_tol tol
-pc_hypre_boomeramg_max_iter it
*/
/*
//ierr = PCSetType(pc, PCASM); PETScErrAct(ierr);
ierr = PCSetType(pc, PCNONE); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCILU); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCBJACOBI); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCLU); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCEISENSTAT); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCSOR); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCJACOBI); PETScErrAct(ierr);
//ierr = PCSetType(pc, PCNONE); PETScErrAct(ierr);
*/
/*
ierr = KSPGetPC(solver, &pc); PETScErrAct(ierr);
ierr = PCSetType(pc, PCILU); PETScErrAct(ierr);
ierr = PCFactorSetLevels(pc, 3); PETScErrAct(ierr);
//ierr = PCFactorSetUseDropTolerance(pc, 1e-3, .1, 50); PETScErrAct(ierr);
//ierr = PCFactorSetFill(pc, 30.7648); PETScErrAct(ierr);
ierr = PCFactorSetReuseOrdering(pc, PETSC_TRUE); PETScErrAct(ierr);
ierr = PCFactorSetReuseFill(pc, PETSC_TRUE); PETScErrAct(ierr);
ierr = PCFactorSetAllowDiagonalFill(pc); PETScErrAct(ierr);
//ierr = PCFactorSetUseInPlace(pc); PETScErrAct(ierr);
*/
ierr = KSPSetInitialGuessNonzero(solver, PETSC_TRUE); PETScErrAct(ierr);
ierr = PetscGetTime(&T1);PETScErrAct(ierr);
ierr = KSPSetFromOptions(solver); PETScErrAct(ierr);
ierr = PetscGetTime(&T2);PETScErrAct(ierr);
PetscPrintf(PCW, "Setup time for the Pressure solver was:%f\n", (T2 - T1));
ierr = KSPView(solver, PETSC_VIEWER_STDOUT_WORLD); PETScErrAct(ierr);
return solver;
}
