MGSolver_Status _GetSolveStatus( MGSolver_PETScData* mgData ) {
PC pc;
const KSPType kspType;
const PCType pcType;
KSPConvergedReason reason;
PetscErrorCode ec;
ec = KSPGetType( mgData->ksp, &kspType );
CheckPETScError( ec );
ec = KSPGetPC( mgData->ksp, &pc );
CheckPETScError( ec );
ec = PCGetType( pc, &pcType );
CheckPETScError( ec );
if( !strcmp( kspType, KSPRICHARDSON ) && !strcmp( pcType, PCSOR ) ) {
double rnorm;
PetscInt curIt;
//rnorm = PETScMatrixSolver_GetResidualNorm( self );
//curIt = PETScMatrixSolver_GetIterations( self );
rnorm = _GetResidualNorm( mgData );
KSPGetIterationNumber( mgData->ksp, &curIt );
//PETScMatrixSolver_SetNormType( self, MultigridSolver_NormType_Preconditioned );
KSPSetNormType( mgData->ksp, MultigridSolver_NormType_Preconditioned );
ec = KSPDefaultConverged( mgData->ksp, curIt, (PetscScalar)rnorm, &reason, PETSC_NULL );
CheckPETScError( ec );
}
else {
ec = KSPGetConvergedReason( mgData->ksp, &reason );
CheckPETScError( ec );
}
return reason;
}
double _GetResidualNorm( MGSolver_PETScData* mgData ) {
PC pc;
const KSPType kspType;
const PCType pcType;
PetscScalar rnorm;
PetscErrorCode ec;
ec = KSPGetType( mgData->ksp, &kspType );
CheckPETScError( ec );
ec = KSPGetPC( mgData->ksp, &pc );
CheckPETScError( ec );
ec = PCGetType( pc, &pcType );
CheckPETScError( ec );
if( !strcmp( kspType, KSPRICHARDSON ) && !strcmp( pcType, PCSOR ) ) {
Vec residual;
//residual = MatrixSolver_GetResidual( mgData );
//rnorm = (PetscScalar)Vector_L2Norm( residual );
residual = _GetResidual( mgData );
VecNorm( residual, NORM_2, &rnorm );
}
else {
ec = KSPGetResidualNorm( mgData->ksp, &rnorm );
CheckPETScError( ec );
}
return (double)rnorm;
}
PetscErrorCode STSetFromOptions_Shell(ST st)
{
PetscErrorCode ierr;
PC pc;
PCType pctype;
KSPType ksptype;
PetscFunctionBegin;
if (!st->ksp) { ierr = STGetKSP(st,&st->ksp);CHKERRQ(ierr); }
ierr = KSPGetPC(st->ksp,&pc);CHKERRQ(ierr);
ierr = KSPGetType(st->ksp,&ksptype);CHKERRQ(ierr);
ierr = PCGetType(pc,&pctype);CHKERRQ(ierr);
if (!pctype && !ksptype) {
if (st->shift_matrix == ST_MATMODE_SHELL) {
/* in shell mode use GMRES with Jacobi as the default */
ierr = KSPSetType(st->ksp,KSPGMRES);CHKERRQ(ierr);
ierr = PCSetType(pc,PCJACOBI);CHKERRQ(ierr);
} else {
/* use direct solver as default */
ierr = KSPSetType(st->ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = PCSetType(pc,PCREDUNDANT);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
static PetscErrorCode PCBDDCScalingSetUp_Deluxe_Private(PC pc)
{
PC_BDDC *pcbddc=(PC_BDDC*)pc->data;
PCBDDCDeluxeScaling deluxe_ctx=pcbddc->deluxe_ctx;
PCBDDCSubSchurs sub_schurs = pcbddc->sub_schurs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!sub_schurs->n_subs) {
PetscFunctionReturn(0);
}
/* Create work vectors for sequential part of deluxe */
ierr = MatCreateVecs(sub_schurs->S_Ej_all,&deluxe_ctx->seq_work1,&deluxe_ctx->seq_work2);CHKERRQ(ierr);
/* Compute deluxe sequential scatter */
if (sub_schurs->reuse_mumps && !sub_schurs->is_dir) {
PCBDDCReuseMumps reuse_mumps = sub_schurs->reuse_mumps;
ierr = PetscObjectReference((PetscObject)reuse_mumps->correction_scatter_B);CHKERRQ(ierr);
deluxe_ctx->seq_scctx = reuse_mumps->correction_scatter_B;
} else {
ierr = VecScatterCreate(pcbddc->work_scaling,sub_schurs->is_Ej_all,deluxe_ctx->seq_work1,NULL,&deluxe_ctx->seq_scctx);CHKERRQ(ierr);
}
/* Create Mat object for deluxe scaling */
ierr = PetscObjectReference((PetscObject)sub_schurs->S_Ej_all);CHKERRQ(ierr);
deluxe_ctx->seq_mat = sub_schurs->S_Ej_all;
if (sub_schurs->sum_S_Ej_all) { /* if this matrix is present, then we need to create the KSP object to invert it */
PC pc_temp;
MatSolverPackage solver=NULL;
char ksp_prefix[256];
size_t len;
ierr = KSPCreate(PETSC_COMM_SELF,&deluxe_ctx->seq_ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(deluxe_ctx->seq_ksp,sub_schurs->sum_S_Ej_all,sub_schurs->sum_S_Ej_all);CHKERRQ(ierr);
ierr = KSPSetType(deluxe_ctx->seq_ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(deluxe_ctx->seq_ksp,&pc_temp);CHKERRQ(ierr);
ierr = PCSetType(pc_temp,PCLU);CHKERRQ(ierr);
ierr = KSPGetPC(pcbddc->ksp_D,&pc_temp);CHKERRQ(ierr);
ierr = PCFactorGetMatSolverPackage(pc_temp,(const MatSolverPackage*)&solver);CHKERRQ(ierr);
if (solver) {
PC new_pc;
PCType type;
ierr = PCGetType(pc_temp,&type);CHKERRQ(ierr);
ierr = KSPGetPC(deluxe_ctx->seq_ksp,&new_pc);CHKERRQ(ierr);
ierr = PCSetType(new_pc,type);CHKERRQ(ierr);
ierr = PCFactorSetMatSolverPackage(new_pc,solver);CHKERRQ(ierr);
}
ierr = PetscStrlen(((PetscObject)(pcbddc->ksp_D))->prefix,&len);CHKERRQ(ierr);
len -= 10; /* remove "dirichlet_" */
ierr = PetscStrncpy(ksp_prefix,((PetscObject)(pcbddc->ksp_D))->prefix,len+1);CHKERRQ(ierr);
ierr = PetscStrcat(ksp_prefix,"deluxe_");CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(deluxe_ctx->seq_ksp,ksp_prefix);CHKERRQ(ierr);
ierr = KSPSetFromOptions(deluxe_ctx->seq_ksp);CHKERRQ(ierr);
ierr = KSPSetUp(deluxe_ctx->seq_ksp);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode BSSCR_pc_warn( PC pc, const char func_name[] )
{
const PCType type;
PCGetType( pc, &type );
if( strcmp(type,PCTYPE_GtKG)!=0 ) {
printf("Warning(%s): PC type (%s) should be gtkg \n",func_name, type );
PetscFunctionReturn(0);
}
PetscFunctionReturn(0);
}
PetscErrorCode BSSCR_BSSCR_pc_error_ScGtKG( PC pc, const char func_name[] )
{
const PCType type;
PCGetType( pc, &type );
if( strcmp(type,PCTYPE_SCGtKG)!=0 ) {
printf("Error(%s): PC type (%s) should be scgtkg. \n",func_name, type );
PetscFinalize();
exit(0);
}
PetscFunctionReturn(0);
}
/*@
PCMGGetSmootherUp - Gets the KSP context to be used as smoother after
coarse grid correction (post-smoother).
Not Collective, KSP returned is parallel if PC is
Input Parameters:
+ pc - the multigrid context
- l - the level (0 is coarsest) to supply
Ouput Parameters:
. ksp - the smoother
Level: advanced
Notes: calling this will result in a different pre and post smoother so you may need to
set options on the pre smoother also
.keywords: MG, multigrid, get, smoother, up, post-smoother, level
.seealso: PCMGGetSmootherUp(), PCMGGetSmootherDown()
@*/
PetscErrorCode PCMGGetSmootherUp(PC pc,PetscInt l,KSP *ksp)
{
PC_MG *mg = (PC_MG*)pc->data;
PC_MG_Levels **mglevels = mg->levels;
PetscErrorCode ierr;
const char *prefix;
MPI_Comm comm;
PetscFunctionBegin;
PetscValidHeaderSpecific(pc,PC_CLASSID,1);
/*
This is called only if user wants a different pre-smoother from post.
Thus we check if a different one has already been allocated,
if not we allocate it.
*/
if (!l) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_OUTOFRANGE,"There is no such thing as a up smoother on the coarse grid");
if (mglevels[l]->smoothu == mglevels[l]->smoothd) {
KSPType ksptype;
PCType pctype;
PC ipc;
PetscReal rtol,abstol,dtol;
PetscInt maxits;
KSPNormType normtype;
ierr = PetscObjectGetComm((PetscObject)mglevels[l]->smoothd,&comm);CHKERRQ(ierr);
ierr = KSPGetOptionsPrefix(mglevels[l]->smoothd,&prefix);CHKERRQ(ierr);
ierr = KSPGetTolerances(mglevels[l]->smoothd,&rtol,&abstol,&dtol,&maxits);CHKERRQ(ierr);
ierr = KSPGetType(mglevels[l]->smoothd,&ksptype);CHKERRQ(ierr);
ierr = KSPGetNormType(mglevels[l]->smoothd,&normtype);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[l]->smoothd,&ipc);CHKERRQ(ierr);
ierr = PCGetType(ipc,&pctype);CHKERRQ(ierr);
ierr = KSPCreate(comm,&mglevels[l]->smoothu);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(mglevels[l]->smoothu,pc->erroriffailure);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)mglevels[l]->smoothu,(PetscObject)pc,mglevels[0]->levels-l);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(mglevels[l]->smoothu,prefix);CHKERRQ(ierr);
ierr = KSPSetTolerances(mglevels[l]->smoothu,rtol,abstol,dtol,maxits);CHKERRQ(ierr);
ierr = KSPSetType(mglevels[l]->smoothu,ksptype);CHKERRQ(ierr);
ierr = KSPSetNormType(mglevels[l]->smoothu,normtype);CHKERRQ(ierr);
ierr = KSPSetConvergenceTest(mglevels[l]->smoothu,KSPConvergedSkip,NULL,NULL);CHKERRQ(ierr);
ierr = KSPGetPC(mglevels[l]->smoothu,&ipc);CHKERRQ(ierr);
ierr = PCSetType(ipc,pctype);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)mglevels[l]->smoothu);CHKERRQ(ierr);
ierr = PetscObjectComposedDataSetInt((PetscObject) mglevels[l]->smoothu, PetscMGLevelId, mglevels[l]->level);CHKERRQ(ierr);
}
if (ksp) *ksp = mglevels[l]->smoothu;
PetscFunctionReturn(0);
}
void SolverLinearPetsc<T>::init ()
{
// Initialize the data structures if not done so already.
if ( !this->initialized() )
{
this->setInitialized( true );
int ierr=0;
// 2.1.x & earlier style
#if (PETSC_VERSION_MAJOR == 2) && (PETSC_VERSION_MINOR <= 1)
// Create the linear solver context
ierr = SLESCreate ( this->worldComm().globalComm(), &M_sles );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Create the Krylov subspace & preconditioner contexts
ierr = SLESGetKSP ( M_sles, &M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
ierr = SLESGetPC ( M_sles, &M_pc );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Have the Krylov subspace method use our good initial guess rather than 0
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_TRUE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set user-specified solver and preconditioner types
this->setPetscSolverType();
this->setPetscPreconditionerType();
this->setPetscConstantNullSpace();
// Set the options from user-input
// Set runtime options, e.g.,
// -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
// These options will override those specified above as long as
// SLESSetFromOptions() is called _after_ any other customization
// routines.
ierr = SLESSetFromOptions ( M_sles );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// 2.2.0 & newer style
#else
// Create the linear solver context
ierr = KSPCreate ( this->worldComm().globalComm(), &M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Create the preconditioner context
ierr = KSPGetPC ( M_ksp, &M_pc );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Have the Krylov subspace method use our good initial guess rather than 0
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_TRUE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
// Set user-specified solver and preconditioner types
this->setPetscSolverType();
this->setPetscConstantNullSpace();
// Set the options from user-input
// Set runtime options, e.g.,
// -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
// These options will override those specified above as long as
// KSPSetFromOptions() is called _after_ any other customization
// routines.
//ierr = PCSetFromOptions ( M_pc );
//CHKERRABORT( this->worldComm().globalComm(),ierr );
ierr = KSPSetFromOptions ( M_ksp );
CHKERRABORT( this->worldComm().globalComm(),ierr );
#endif
// Have the Krylov subspace method use our good initial guess
// rather than 0, unless the user requested a KSPType of
// preonly, which complains if asked to use initial guesses.
#if PETSC_VERSION_LESS_THAN(3,0,0)
KSPType ksp_type;
#else
#if PETSC_VERSION_LESS_THAN(3,4,0)
const KSPType ksp_type;
#else
KSPType ksp_type;
#endif
#endif
ierr = KSPGetType ( M_ksp, &ksp_type );
CHKERRABORT( this->worldComm().globalComm(),ierr );
if ( std::string((char*)ksp_type) == std::string( ( char* )KSPPREONLY ) )
{
ierr = KSPSetInitialGuessNonzero ( M_ksp, PETSC_FALSE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
}
if ( std::string((char*)ksp_type) == std::string( ( char* )KSPGMRES ) )
{
int nRestartGMRES = option(_name="gmres-restart", _prefix=this->prefix() ).template as<int>();
ierr = KSPGMRESSetRestart( M_ksp, nRestartGMRES );
CHKERRABORT( this->worldComm().globalComm(),ierr );
}
// Notify PETSc of location to store residual history.
// This needs to be called before any solves, since
// it sets the residual history length to zero. The default
// behavior is for PETSc to allocate (internally) an array
// of size 1000 to hold the residual norm history.
ierr = KSPSetResidualHistory( M_ksp,
PETSC_NULL, // pointer to the array which holds the history
PETSC_DECIDE, // size of the array holding the history
PETSC_TRUE ); // Whether or not to reset the history for each solve.
CHKERRABORT( this->worldComm().globalComm(),ierr );
//If there is a preconditioner object we need to set the internal setup and apply routines
if ( this->M_preconditioner )
{
VLOG(2) << "preconditioner: " << this->M_preconditioner << "\n";
PCSetType(M_pc, PCSHELL);
PCShellSetName( M_pc, this->M_preconditioner->name().c_str() );
PCShellSetContext( M_pc,( void* )this->M_preconditioner.get() );
PCShellSetSetUp( M_pc,__feel_petsc_preconditioner_setup );
PCShellSetApply( M_pc,__feel_petsc_preconditioner_apply );
PCShellSetView( M_pc,__feel_petsc_preconditioner_view );
#if PETSC_VERSION_LESS_THAN(3,4,0)
const PCType pc_type;
#else
PCType pc_type;
#endif
ierr = PCGetType ( M_pc, &pc_type );
CHKERRABORT( this->worldComm().globalComm(),ierr );
VLOG(2) << "preconditioner set as " << pc_type << "\n";
}
else
{
this->setPetscPreconditionerType();
// sets the software that is used to perform the factorization
PetscPCFactorSetMatSolverPackage( M_pc,this->matSolverPackageType() );
}
if ( Environment::vm(_name="ksp-monitor",_prefix=this->prefix()).template as<bool>() )
{
KSPMonitorSet( M_ksp,KSPMonitorDefault,PETSC_NULL,PETSC_NULL );
}
}
}
PetscErrorCode TSSetUp_Sundials_Nonlinear(TS ts)
{
TS_Sundials *cvode = (TS_Sundials*)ts->data;
PetscErrorCode ierr;
PetscInt glosize,locsize,i,flag;
PetscScalar *y_data,*parray;
void *mem;
const PCType pctype;
PetscTruth pcnone;
Vec sol = ts->vec_sol;
PetscFunctionBegin;
ierr = PCSetFromOptions(cvode->pc);CHKERRQ(ierr);
/* get the vector size */
ierr = VecGetSize(ts->vec_sol,&glosize);CHKERRQ(ierr);
ierr = VecGetLocalSize(ts->vec_sol,&locsize);CHKERRQ(ierr);
/* allocate the memory for N_Vec y */
cvode->y = N_VNew_Parallel(cvode->comm_sundials,locsize,glosize);
if (!cvode->y) SETERRQ(1,"cvode->y is not allocated");
/* initialize N_Vec y: copy ts->vec_sol to cvode->y */
ierr = VecGetArray(ts->vec_sol,&parray);CHKERRQ(ierr);
y_data = (PetscScalar *) N_VGetArrayPointer(cvode->y);
for (i = 0; i < locsize; i++) y_data[i] = parray[i];
/*ierr = PetscMemcpy(y_data,parray,locsize*sizeof(PETSC_SCALAR)); CHKERRQ(ierr);*/
ierr = VecRestoreArray(ts->vec_sol,PETSC_NULL);CHKERRQ(ierr);
ierr = VecDuplicate(ts->vec_sol,&cvode->update);CHKERRQ(ierr);
ierr = VecDuplicate(ts->vec_sol,&cvode->func);CHKERRQ(ierr);
ierr = PetscLogObjectParent(ts,cvode->update);CHKERRQ(ierr);
ierr = PetscLogObjectParent(ts,cvode->func);CHKERRQ(ierr);
/*
Create work vectors for the TSPSolve_Sundials() routine. Note these are
allocated with zero space arrays because the actual array space is provided
by Sundials and set using VecPlaceArray().
*/
ierr = VecCreateMPIWithArray(((PetscObject)ts)->comm,locsize,PETSC_DECIDE,0,&cvode->w1);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(((PetscObject)ts)->comm,locsize,PETSC_DECIDE,0,&cvode->w2);CHKERRQ(ierr);
ierr = PetscLogObjectParent(ts,cvode->w1);CHKERRQ(ierr);
ierr = PetscLogObjectParent(ts,cvode->w2);CHKERRQ(ierr);
/* Call CVodeCreate to create the solver memory and the use of a Newton iteration */
mem = CVodeCreate(cvode->cvode_type, CV_NEWTON);
if (!mem) SETERRQ(PETSC_ERR_MEM,"CVodeCreate() fails");
cvode->mem = mem;
/* Set the pointer to user-defined data */
flag = CVodeSetUserData(mem, ts);
if (flag) SETERRQ(PETSC_ERR_LIB,"CVodeSetUserData() fails");
/* Call CVodeInit to initialize the integrator memory and specify the
* user's right hand side function in u'=f(t,u), the inital time T0, and
* the initial dependent variable vector cvode->y */
flag = CVodeInit(mem,TSFunction_Sundials,ts->ptime,cvode->y);
if (flag){
SETERRQ1(PETSC_ERR_LIB,"CVodeInit() fails, flag %d",flag);
}
flag = CVodeSStolerances(mem,cvode->reltol,cvode->abstol);
if (flag){
SETERRQ1(PETSC_ERR_LIB,"CVodeSStolerances() fails, flag %d",flag);
}
/* initialize the number of steps */
ierr = TSMonitor(ts,ts->steps,ts->ptime,sol);CHKERRQ(ierr);
/* call CVSpgmr to use GMRES as the linear solver. */
/* setup the ode integrator with the given preconditioner */
ierr = PCGetType(cvode->pc,&pctype);CHKERRQ(ierr);
ierr = PetscTypeCompare((PetscObject)cvode->pc,PCNONE,&pcnone);CHKERRQ(ierr);
if (pcnone){
flag = CVSpgmr(mem,PREC_NONE,0);
if (flag) SETERRQ1(PETSC_ERR_LIB,"CVSpgmr() fails, flag %d",flag);
} else {
flag = CVSpgmr(mem,PREC_LEFT,0);
if (flag) SETERRQ1(PETSC_ERR_LIB,"CVSpgmr() fails, flag %d",flag);
/* Set preconditioner and solve routines Precond and PSolve,
and the pointer to the user-defined block data */
flag = CVSpilsSetPreconditioner(mem,TSPrecond_Sundials,TSPSolve_Sundials);
if (flag) SETERRQ1(PETSC_ERR_LIB,"CVSpilsSetPreconditioner() fails, flag %d", flag);
}
flag = CVSpilsSetGSType(mem, MODIFIED_GS);
if (flag) SETERRQ1(PETSC_ERR_LIB,"CVSpgmrSetGSType() fails, flag %d",flag);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A; /* linear system matrix */
PetscErrorCode ierr;
PetscMPIInt rank=0;
PetscBool flg;
PetscViewer fd; /* viewer */
PetscViewer log;
char file[PETSC_MAX_PATH_LEN];
char logfile[PETSC_MAX_PATH_LEN];
char lockfile[PETSC_MAX_PATH_LEN], tmpstr[PETSC_MAX_PATH_LEN], dirname[PETSC_MAX_PATH_LEN], matrix[PETSC_MAX_PATH_LEN];
char hash[20];
PetscLogDouble solveTime,endTime,startTime;
PetscInt its;
PetscReal norm;
KSP ksp; // Linear solver context
Vec b,x,u; // RHS, solution, vector for norm calculation
PetscScalar one = 1.0;
PetscInt m, n, i;
FILE *lock;
/*
if (rank == 0) {
printf("Command line arguments:\n");
for (i=0; i < argc; i++)
printf("%d: %s\n", i, args[i]);
}
// Save args
int argcount = argc;
char **argv = (char**) malloc (argc*sizeof(char*));
for (i=0; i < argc; i++) {
argv[i] = (char*) malloc(strlen(args[i]) + 1);
strcpy(argv[i],args[i]);
}
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
*/
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-hash",hash,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) {
strcpy(hash,"nohash");
}
ierr = PetscOptionsGetString(PETSC_NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) {
PetscPrintf(PETSC_COMM_WORLD,"Must indicate matrix file with the -f option");
}
/* Create lock file */
if (rank == 0) {
for (i = strlen(file); i> 0; i--) if (file[i] == '.') break;
strncpy(tmpstr, file, i-1);
for (i = strlen(tmpstr); i> 0; i--) if (file[i] == '/') break;
strncpy(dirname, tmpstr, i);
dirname[i] = '\0';
sprintf(lockfile,"%s/../timing/.%s.%s", dirname, basename(tmpstr), hash);
sprintf(logfile,"%s/../timing/%s.%s.log", dirname, basename(tmpstr), hash);
lock = fopen(lockfile, "w");
fprintf(lock, "%s\n", file);
fclose(lock);
}
/* Read file */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
// Create matrix
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetType(A,MATMPIAIJ); CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
// Load matrix from file
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
ierr = MatGetSize(A, &m, &n); CHKERRQ(ierr);
// Assemble matrix
//ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
//ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
// Create RHS vector
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,PETSC_DECIDE,n); CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
ierr = VecSet(b,one); CHKERRQ(ierr);
//ierr = VecLoad(b,fd);CHKERRQ(ierr);
// Create vectors x and u
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&u);CHKERRQ(ierr);
// Create KSP
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp); CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp,PETSC_FALSE);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr);
// Setup KSP
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
ierr = KSPSetUpOnBlocks(ksp);CHKERRQ(ierr);
// Get start time
ierr = PetscTime(&startTime);CHKERRQ(ierr);
// Get KSP and PC type
KSPType kt;
ierr = KSPGetType(ksp,&kt);
PC pc;
ierr = KSPGetPC(ksp,&pc);
PCType pt;
ierr = PCGetType(pc,&pt);
// Print method info
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, logfile, &log); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(log, "Hash: %s\n", hash);
ierr = PetscViewerASCIIPrintf(log, "%s | %s",kt,pt);CHKERRQ(ierr);
// Make sure the program doesn't crash
// while trying to solve the system
PetscPushErrorHandler(PetscIgnoreErrorHandler,NULL);
ierr = KSPSolve(ksp,b,x);
PetscPopErrorHandler();
// Check if anything went wrong
if(ierr == 0 || ierr == -1){
// If no error occurred or stopped by MyKSPMonitor,
// compute normal and stuff
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = MatMult(A,x,u);CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscTime(&endTime);CHKERRQ(ierr);
// Compute solve time
solveTime = endTime - startTime;
// Check if KSP converged
KSPConvergedReason reason;
KSPGetConvergedReason(ksp,&reason);
// Print convergence code, solve time, preconditioned norm, iterations
ierr = PetscViewerASCIIPrintf(log, " | %D | %e | %g | %D\n",reason,solveTime,norm,its);CHKERRQ(ierr);
ierr = KSPView(ksp,log);
ierr = PCView(pc,log);
ierr = PetscLogView(log);
}
else{
// Disaster happened, bail out
if (rank == 0) remove(lockfile);
PetscFinalize();
return 0;
}
// Again, destroy KSP and vector
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
if (rank == 0) remove(lockfile);
PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat mat; /* matrix */
Vec b,ustar,u; /* vectors (RHS, exact solution, approx solution) */
PC pc; /* PC context */
KSP ksp; /* KSP context */
PetscErrorCode ierr;
PetscInt n = 10,i,its,col[3];
PetscScalar value[3];
PCType pcname;
KSPType kspname;
PetscReal norm,tol=1000.*PETSC_MACHINE_EPSILON;
PetscInitialize(&argc,&args,(char*)0,help);
/* Create and initialize vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&b);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&ustar);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&u);CHKERRQ(ierr);
ierr = VecSet(ustar,1.0);CHKERRQ(ierr);
ierr = VecSet(u,0.0);CHKERRQ(ierr);
/* Create and assemble matrix */
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,n,n,3,NULL,&mat);CHKERRQ(ierr);
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(mat,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0] = n - 2; col[1] = n - 1;
ierr = MatSetValues(mat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0; col[0] = 0; col[1] = 1; value[0] = 2.0; value[1] = -1.0;
ierr = MatSetValues(mat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Compute right-hand-side vector */
ierr = MatMult(mat,ustar,b);CHKERRQ(ierr);
/* Create PC context and set up data structures */
ierr = PCCreate(PETSC_COMM_WORLD,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE);CHKERRQ(ierr);
ierr = PCSetFromOptions(pc);CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat);CHKERRQ(ierr);
ierr = PCSetUp(pc);CHKERRQ(ierr);
/* Create KSP context and set up data structures */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPRICHARDSON);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat);CHKERRQ(ierr);
ierr = KSPSetPC(ksp,pc);CHKERRQ(ierr);
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
/* Solve the problem */
ierr = KSPGetType(ksp,&kspname);CHKERRQ(ierr);
ierr = PCGetType(pc,&pcname);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"Running %s with %s preconditioning\n",kspname,pcname);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b,u);CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,ustar);CHKERRQ(ierr);
ierr = VecNorm(u,NORM_2,&norm);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"2 norm of error %g Number of iterations %D\n",(double)norm,its);CHKERRQ(ierr);
}
/* Free data structures */
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&ustar);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = PCDestroy(&pc);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc, char *argv[])
{
// Initialize libMesh
libMesh::LibMeshInit init(argc, argv);
libMesh::Parallel::Communicator& WorldComm = init.comm();
libMesh::PetscMatrix<libMesh::Number> matrix_A(WorldComm);
matrix_A.init(4,4,4,4);
matrix_A.set(0,0,1.);
// matrix_A.set(0,1,2.);
// matrix_A.set(0,2,3.);
// matrix_A.set(0,3,4.);
// matrix_A.set(1,0,2.);
matrix_A.set(1,1,5.);
// matrix_A.set(1,2,3.);
// matrix_A.set(1,3,7.);
// matrix_A.set(2,0,3.);
// matrix_A.set(2,1,3.);
matrix_A.set(2,2,9.);
// matrix_A.set(2,3,6.);
// matrix_A.set(3,0,4.);
// matrix_A.set(3,1,7.);
// matrix_A.set(3,2,6.);
matrix_A.set(3,3,1.);
matrix_A.close();
Mat dummy_inv_A;
MatCreate(PETSC_COMM_WORLD,&dummy_inv_A);
MatSetType(dummy_inv_A,MATMPIAIJ);
MatSetSizes(dummy_inv_A,PETSC_DECIDE,PETSC_DECIDE,4,4);
MatMPIAIJSetPreallocation(dummy_inv_A,2,NULL,0,NULL);
MatSetUp(dummy_inv_A);
// Dummy matrices
// Mat dummy_A, dummy_inv_A;
//
//
libMesh::PetscVector<libMesh::Number> vector_unity(WorldComm,4,4);
libMesh::PetscVector<libMesh::Number> vector_dummy_answer(WorldComm,4,4);
VecSet(vector_unity.vec(),1);
vector_unity.close();
VecSet(vector_dummy_answer.vec(),0);
vector_dummy_answer.close();
// Solver
// libMesh::PetscLinearSolver<libMesh::Number> KSP_dummy_solver(WorldComm);
// KSP_dummy_solver.init(&matrix_A);
// KSPSetOperators(KSP_dummy_solver.ksp(),matrix_A.mat(),NULL);
KSP ksp;
PC pc;
KSPCreate(PETSC_COMM_WORLD,&ksp);
KSPSetOperators(ksp, matrix_A.mat(), matrix_A.mat());
KSPGetPC(ksp,&pc);
PCSetFromOptions(pc);
PCType dummy_type;
PCGetType(pc,&dummy_type);
std::cout << std::endl << dummy_type << std::endl << std::endl;
// PCSetType(pc,PCSPAI);
// PCHYPRESetType(pc,"parasails");
KSPSetUp(ksp);
KSPSolve(ksp,vector_unity.vec(),vector_dummy_answer.vec());
PCComputeExplicitOperator(pc,&dummy_inv_A);
// KSPGetOperators(KSP_dummy_solver.ksp(),&dummy_A,&dummy_inv_A);
libMesh::PetscMatrix<libMesh::Number> matrix_invA(dummy_inv_A,WorldComm);
matrix_invA.close();
//
// // KSP_dummy_solver.solve(matrix_A,vector_dummy_answer,vector_unity,1E-5,10000);
//
// vector_dummy_answer.print_matlab();
//
// libMesh::PetscMatrix<libMesh::Number> product_mat(WorldComm);
matrix_A.print_matlab();
matrix_invA.print_matlab();
vector_dummy_answer.print_matlab();
return 0;
}
void PETScLinearSolver::Solver()
{
//TEST
#ifdef TEST_MEM_PETSC
PetscLogDouble mem1, mem2;
PetscMemoryGetCurrentUsage(&mem1);
#endif
/*
//TEST
PetscViewer viewer;
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "x.txt", &viewer);
PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);
PetscObjectSetName((PetscObject)x,"Solution");
VecView(x, viewer);
*/
int its;
PetscLogDouble v1,v2;
KSPConvergedReason reason;
// #define PETSC34
//kg44 quick fix to compile PETSC with version PETSCV3.4
#ifdef USEPETSC34
PetscTime(&v1);
#else
PetscGetTime(&v1);
#endif
#if (PETSC_VERSION_MAJOR == 3) && (PETSC_VERSION_MINOR > 4)
KSPSetOperators(lsolver, A, A);
#else
KSPSetOperators(lsolver, A, A, DIFFERENT_NONZERO_PATTERN);
#endif
KSPSolve(lsolver, b, x);
KSPGetConvergedReason(lsolver,&reason); //CHKERRQ(ierr);
if (reason==KSP_DIVERGED_INDEFINITE_PC)
{
PetscPrintf(PETSC_COMM_WORLD,"\nDivergence because of indefinite preconditioner;\n");
PetscPrintf(PETSC_COMM_WORLD,"Run the executable again but with -pc_factor_shift_positive_definite option.\n");
}
else if (reason<0)
{
PetscPrintf(PETSC_COMM_WORLD,"\nOther kind of divergence: this should not happen.\n");
}
else
{
const char *slv_type;
const char *prc_type;
KSPGetType(lsolver, &slv_type);
PCGetType(prec, &prc_type);
PetscPrintf(PETSC_COMM_WORLD,"\n================================================");
PetscPrintf(PETSC_COMM_WORLD, "\nLinear solver %s with %s preconditioner",
slv_type, prc_type);
KSPGetIterationNumber(lsolver,&its); //CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"\nConvergence in %d iterations.\n",(int)its);
PetscPrintf(PETSC_COMM_WORLD,"\n================================================");
}
PetscPrintf(PETSC_COMM_WORLD,"\n");
//VecAssemblyBegin(x);
//VecAssemblyEnd(x);
//kg44 quick fix to compile PETSC with version PETSCV3.4
#ifdef USEPETSC34
PetscTime(&v2);
#else
PetscGetTime(&v2);
#endif
time_elapsed += v2-v1;
#define aTEST_OUT
#ifdef TEST_OUT
//TEST
PetscViewer viewer;
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "x2.txt", &viewer);
PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);
PetscObjectSetName((PetscObject)A,"Matrix");
MatView(A, viewer);
PetscObjectSetName((PetscObject)x,"Solution");
VecView(x, viewer);
PetscObjectSetName((PetscObject)b,"RHS");
VecView(b, viewer);
VecDestroy(&b);
VecDestroy(&x);
MatDestroy(&A);
if(lsolver) KSPDestroy(&lsolver);
// if(prec) PCDestroy(&prec);
if(global_x0)
delete [] global_x0;
if(global_x1)
delete [] global_x1;
PetscFinalize();
exit(0);
#endif
#ifdef TEST_MEM_PETSC
//TEST
PetscMemoryGetCurrentUsage(&mem2);
PetscPrintf(PETSC_COMM_WORLD, "###Memory usage by solver. Before :%f After:%f Increase:%d\n", mem1, mem2, (int)(mem2 - mem1));
#endif
}
std::pair<std::string,std::string>
RBConstructionBase<Base>::set_alternative_solver
(AutoPtr<LinearSolver<Number> >&
#ifdef LIBMESH_HAVE_PETSC
ls
#endif
)
{
// It seems that setting it this generic way has no effect...
// PreconditionerType orig_pc = this->linear_solver->preconditioner_type();
// this->linear_solver->set_preconditioner_type(AMG_PRECOND);
// so we do it the "hard" way.
std::string orig_petsc_pc_type_string, orig_petsc_ksp_type_string;
#ifdef LIBMESH_HAVE_PETSC
// ... but we can set it the "hard" way
PetscLinearSolver<Number>* petsc_linear_solver =
libmesh_cast_ptr<PetscLinearSolver<Number>*>(ls.get());
// Note: #define PCType char*, and PCGetType just sets a pointer. We'll use
// the string below to make a real copy, and set the PC back to its original
// type at the end of the function.
#if PETSC_VERSION_LESS_THAN(3,0,0) || !PETSC_VERSION_LESS_THAN(3,4,0)
// Pre-3.0 and petsc-dev (as of October 2012) use non-const versions
PCType orig_petsc_pc_type;
KSPType orig_petsc_ksp_type;
#else
const PCType orig_petsc_pc_type;
const KSPType orig_petsc_ksp_type;
#endif
if (petsc_linear_solver)
{
PC pc = petsc_linear_solver->pc();
int ierr = PCGetType(pc, &orig_petsc_pc_type);
LIBMESH_CHKERRABORT(ierr);
KSP ksp = petsc_linear_solver->ksp();
ierr = KSPGetType(ksp, &orig_petsc_ksp_type);
LIBMESH_CHKERRABORT(ierr);
// libMesh::out << "orig_petsc_pc_type (before)=" << orig_petsc_pc_type << std::endl;
// Make actual copies of the original PC and KSP types
orig_petsc_pc_type_string = orig_petsc_pc_type;
orig_petsc_ksp_type_string = orig_petsc_ksp_type;
#ifdef LIBMESH_HAVE_PETSC_HYPRE
// Set solver/PC combo specified in input file...
if (this->alternative_solver == "amg")
{
// Set HYPRE and boomeramg PC types
ierr = PCSetType(pc, PCHYPRE);
LIBMESH_CHKERRABORT(ierr);
ierr = PCHYPRESetType(pc, "boomeramg");
LIBMESH_CHKERRABORT(ierr);
}
#endif // LIBMESH_HAVE_PETSC_HYPRE
if (this->alternative_solver == "mumps")
{
// We'll use MUMPS... TODO: configure libmesh to detect
// when MUMPS is available via PETSc.
// No initial guesses can be specified with KSPPREONLY. We
// can leave the solver as gmres or whatever and it should
// converge in 1 iteration. Otherwise, to use KSPPREONLY,
// you may need to do:
// KSPSetInitialGuessNonzero(ksp, PETSC_FALSE);
// ierr = KSPSetType(ksp, KSPPREONLY);
// LIBMESH_CHKERRABORT(ierr);
// Need to call the equivalent for the command line options:
// -ksp_type preonly -pc_type lu -pc_factor_mat_solver_package mumps
ierr = PCSetType(pc, PCLU);
LIBMESH_CHKERRABORT(ierr);
#if !(PETSC_VERSION_LESS_THAN(3,0,0))
ierr = PCFactorSetMatSolverPackage(pc,"mumps");
LIBMESH_CHKERRABORT(ierr);
#endif
}
}
else
{
// Otherwise, the cast failed and we are not using PETSc...
libMesh::out << "You are not using PETSc, so don't know how to set AMG PC." << std::endl;
libMesh::out << "Returning empty string!" << std::endl;
}
#endif // LIBMESH_HAVE_PETSC
return std::make_pair(orig_petsc_pc_type_string, orig_petsc_ksp_type_string);
}
PetscErrorCode PCBDDCSubSchursSetUp(PCBDDCSubSchurs sub_schurs, Mat S, IS is_A_I, IS is_A_B, PetscInt ncc, IS is_cc[], PetscInt xadj[], PetscInt adjncy[], PetscInt nlayers)
{
Mat A_II,A_IB,A_BI,A_BB;
ISLocalToGlobalMapping BtoNmap,ItoNmap;
PetscBT touched;
PetscInt i,n_I,n_B,n_local,*local_numbering;
PetscBool is_sorted;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = ISSorted(is_A_I,&is_sorted);CHKERRQ(ierr);
if (!is_sorted) {
SETERRQ(PetscObjectComm((PetscObject)is_A_I),PETSC_ERR_PLIB,"IS for I dofs should be shorted");
}
ierr = ISSorted(is_A_B,&is_sorted);CHKERRQ(ierr);
if (!is_sorted) {
SETERRQ(PetscObjectComm((PetscObject)is_A_B),PETSC_ERR_PLIB,"IS for B dofs should be shorted");
}
/* get sizes */
ierr = ISGetLocalSize(is_A_I,&n_I);CHKERRQ(ierr);
ierr = ISGetLocalSize(is_A_B,&n_B);CHKERRQ(ierr);
n_local = n_I+n_B;
/* maps */
ierr = ISLocalToGlobalMappingCreateIS(is_A_B,&BtoNmap);CHKERRQ(ierr);
if (nlayers >= 0 && xadj != NULL && adjncy != NULL) { /* I problems have a different size of the original ones */
ierr = ISLocalToGlobalMappingCreateIS(is_A_I,&ItoNmap);CHKERRQ(ierr);
/* allocate some auxiliary space */
ierr = PetscMalloc1(n_local,&local_numbering);CHKERRQ(ierr);
ierr = PetscBTCreate(n_local,&touched);CHKERRQ(ierr);
} else {
ItoNmap = 0;
local_numbering = 0;
touched = 0;
}
/* get Schur complement matrices */
ierr = MatSchurComplementGetSubMatrices(S,&A_II,NULL,&A_IB,&A_BI,&A_BB);CHKERRQ(ierr);
/* allocate space for schur complements */
ierr = PetscMalloc5(ncc,&sub_schurs->is_AEj_I,ncc,&sub_schurs->is_AEj_B,ncc,&sub_schurs->S_Ej,ncc,&sub_schurs->work1,ncc,&sub_schurs->work2);CHKERRQ(ierr);
sub_schurs->n_subs = ncc;
/* cycle on subsets and extract schur complements */
for (i=0;i<sub_schurs->n_subs;i++) {
Mat AE_II,AE_IE,AE_EI,AE_EE;
IS is_I,is_subset_B;
/* get IS for subsets in B numbering */
ierr = ISDuplicate(is_cc[i],&sub_schurs->is_AEj_B[i]);CHKERRQ(ierr);
ierr = ISSort(sub_schurs->is_AEj_B[i]);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyIS(BtoNmap,IS_GTOLM_DROP,sub_schurs->is_AEj_B[i],&is_subset_B);CHKERRQ(ierr);
/* BB block on subset */
ierr = MatGetSubMatrix(A_BB,is_subset_B,is_subset_B,MAT_INITIAL_MATRIX,&AE_EE);CHKERRQ(ierr);
if (ItoNmap) { /* is ItoNmap has been computed, extracts only a part of I dofs */
const PetscInt* idx_B;
PetscInt n_local_dofs,n_prev_added,j,layer,subset_size;
/* all boundary dofs must be skipped when adding layers */
ierr = PetscBTMemzero(n_local,touched);CHKERRQ(ierr);
ierr = ISGetIndices(is_A_B,&idx_B);CHKERRQ(ierr);
for (j=0;j<n_B;j++) {
ierr = PetscBTSet(touched,idx_B[j]);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(is_A_B,&idx_B);CHKERRQ(ierr);
/* add next layers of dofs */
ierr = ISGetLocalSize(is_cc[i],&subset_size);CHKERRQ(ierr);
ierr = ISGetIndices(is_cc[i],&idx_B);CHKERRQ(ierr);
ierr = PetscMemcpy(local_numbering,idx_B,subset_size*sizeof(PetscInt));CHKERRQ(ierr);
ierr = ISRestoreIndices(is_cc[i],&idx_B);CHKERRQ(ierr);
n_local_dofs = subset_size;
n_prev_added = subset_size;
for (layer=0;layer<nlayers;layer++) {
PetscInt n_added;
if (n_local_dofs == n_I+subset_size) break;
if (n_local_dofs > n_I+subset_size) {
SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Error querying layer %d. Out of bound access (%d > %d)",layer,n_local_dofs,n_I+subset_size);
}
ierr = PCBDDCAdjGetNextLayer_Private(local_numbering+n_local_dofs,n_prev_added,touched,xadj,adjncy,&n_added);CHKERRQ(ierr);
n_prev_added = n_added;
n_local_dofs += n_added;
if (!n_added) break;
}
/* IS for I dofs in original numbering and in I numbering */
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)ItoNmap),n_local_dofs-subset_size,local_numbering+subset_size,PETSC_COPY_VALUES,&sub_schurs->is_AEj_I[i]);CHKERRQ(ierr);
ierr = ISSort(sub_schurs->is_AEj_I[i]);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyIS(ItoNmap,IS_GTOLM_DROP,sub_schurs->is_AEj_I[i],&is_I);CHKERRQ(ierr);
/* II block */
ierr = MatGetSubMatrix(A_II,is_I,is_I,MAT_INITIAL_MATRIX,&AE_II);CHKERRQ(ierr);
} else { /* in this case we can take references of already existing IS and matrices for I dofs */
/* IS for I dofs in original numbering */
ierr = PetscObjectReference((PetscObject)is_A_I);CHKERRQ(ierr);
sub_schurs->is_AEj_I[i] = is_A_I;
/* IS for I dofs in I numbering TODO: "first" argument of ISCreateStride is not general */
ierr = ISCreateStride(PetscObjectComm((PetscObject)is_A_I),n_I,0,1,&is_I);CHKERRQ(ierr);
/* II block is the same */
ierr = PetscObjectReference((PetscObject)A_II);CHKERRQ(ierr);
AE_II = A_II;
}
/* IE block */
ierr = MatGetSubMatrix(A_IB,is_I,is_subset_B,MAT_INITIAL_MATRIX,&AE_IE);CHKERRQ(ierr);
/* EI block */
ierr = MatGetSubMatrix(A_BI,is_subset_B,is_I,MAT_INITIAL_MATRIX,&AE_EI);CHKERRQ(ierr);
/* setup Schur complements on subset */
ierr = MatCreateSchurComplement(AE_II,AE_II,AE_IE,AE_EI,AE_EE,&sub_schurs->S_Ej[i]);CHKERRQ(ierr);
ierr = MatGetVecs(sub_schurs->S_Ej[i],&sub_schurs->work1[i],&sub_schurs->work2[i]);CHKERRQ(ierr);
if (AE_II == A_II) { /* we can reuse the same ksp */
KSP ksp;
ierr = MatSchurComplementGetKSP(S,&ksp);CHKERRQ(ierr);
ierr = MatSchurComplementSetKSP(sub_schurs->S_Ej[i],ksp);CHKERRQ(ierr);
} else { /* build new ksp object which inherits ksp and pc types from the original one */
KSP origksp,schurksp;
PC origpc,schurpc;
KSPType ksp_type;
PCType pc_type;
PetscInt n_internal;
ierr = MatSchurComplementGetKSP(S,&origksp);CHKERRQ(ierr);
ierr = MatSchurComplementGetKSP(sub_schurs->S_Ej[i],&schurksp);CHKERRQ(ierr);
ierr = KSPGetType(origksp,&ksp_type);CHKERRQ(ierr);
ierr = KSPSetType(schurksp,ksp_type);CHKERRQ(ierr);
ierr = KSPGetPC(schurksp,&schurpc);CHKERRQ(ierr);
ierr = KSPGetPC(origksp,&origpc);CHKERRQ(ierr);
ierr = PCGetType(origpc,&pc_type);CHKERRQ(ierr);
ierr = PCSetType(schurpc,pc_type);CHKERRQ(ierr);
ierr = ISGetSize(is_I,&n_internal);CHKERRQ(ierr);
if (n_internal) { /* UMFPACK gives error with 0 sized problems */
MatSolverPackage solver=NULL;
ierr = PCFactorGetMatSolverPackage(origpc,(const MatSolverPackage*)&solver);CHKERRQ(ierr);
if (solver) {
ierr = PCFactorSetMatSolverPackage(schurpc,solver);CHKERRQ(ierr);
}
}
ierr = KSPSetUp(schurksp);CHKERRQ(ierr);
}
/* free */
ierr = MatDestroy(&AE_II);CHKERRQ(ierr);
ierr = MatDestroy(&AE_EE);CHKERRQ(ierr);
ierr = MatDestroy(&AE_IE);CHKERRQ(ierr);
ierr = MatDestroy(&AE_EI);CHKERRQ(ierr);
ierr = ISDestroy(&is_I);CHKERRQ(ierr);
ierr = ISDestroy(&is_subset_B);CHKERRQ(ierr);
}
/* free */
ierr = ISLocalToGlobalMappingDestroy(&ItoNmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&BtoNmap);CHKERRQ(ierr);
ierr = PetscFree(local_numbering);CHKERRQ(ierr);
ierr = PetscBTDestroy(&touched);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode efs_setup(efs *slv, int offset[], int stride[])
{
PetscErrorCode ierr;
PetscInt xs, ys, zs, xm, ym, zm;
PCType pc_type;
if (efs_log(slv, EFS_LOG_STATUS)) {
ierr = ef_io_print(slv->comm, "Setting up electric field solver");CHKERRQ(ierr);
}
slv->ts = 0;
if (efs_log(slv, EFS_LOG_RESIDUAL)) {
ierr = PetscOptionsSetValue("-ksp_monitor_short", NULL);CHKERRQ(ierr);
}
ierr = DMDASetFieldName(slv->dm, 0,"potential");CHKERRQ(ierr);
if (slv->grid.nd == 2) {
ierr = DMDAGetCorners(slv->dm, &xs, &ys, 0, &xm, &ym, 0);CHKERRQ(ierr);
slv->dmap = ef_dmap_create_2d(xs - offset[0], ys - offset[1], xm, ym, stride);
} else if (slv->grid.nd == 3) {
ierr = DMDAGetCorners(slv->dm, &xs, &ys, &zs, &xm, &ym, &zm);CHKERRQ(ierr);
slv->dmap = ef_dmap_create_3d(xs - offset[0], ys - offset[1], zs - offset[2], xm, ym, zm, stride);
} else {
SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_SUP, "Unsupported dimmension: %d", slv->grid.nd);
}
ierr = ef_callback_create(&slv->callback);CHKERRQ(ierr);
ierr = KSPCreate(slv->comm, &slv->ksp);CHKERRQ(ierr);
if (efs_log(slv, EFS_LOG_EIGS)) {
ierr = KSPSetComputeEigenvalues(slv->ksp, PETSC_TRUE);CHKERRQ(ierr);
}
ierr = KSPSetDM(slv->ksp, slv->dm);CHKERRQ(ierr);
ierr = KSPGetPC(slv->ksp, &slv->pc);CHKERRQ(ierr);
ierr = PCSetType(slv->pc, PCMG);CHKERRQ(ierr);
if (slv->options.galerkin) {
ierr = PCMGSetGalerkin(slv->pc, PETSC_TRUE);CHKERRQ(ierr);
} else {
ierr = PCMGSetGalerkin(slv->pc, PETSC_FALSE);CHKERRQ(ierr);
}
ierr = KSPSetComputeOperators(slv->ksp, slv->callback->matrix, slv);CHKERRQ(ierr);
ierr = KSPSetComputeRHS(slv->ksp, slv->callback->rhs, slv);CHKERRQ(ierr);
ierr = KSPSetComputeInitialGuess(slv->ksp, slv->callback->guess, slv);CHKERRQ(ierr);
ierr = KSPSetFromOptions(slv->ksp);CHKERRQ(ierr);
ierr = PCGetType(slv->pc, &pc_type);CHKERRQ(ierr);
ierr = PCMGGetLevels(slv->pc, &slv->options.levels);CHKERRQ(ierr);
if (slv->options.levels < 1) slv->options.levels++;
ierr = PCMGGetGalerkin(slv->pc, &slv->options.galerkin);CHKERRQ(ierr);
if (strcmp(pc_type, PCGAMG) == 0
|| strcmp(pc_type, PCHYPRE) == 0) slv->options.galerkin = 1;
if (!slv->options.galerkin) {
slv->levels = (ef_level*) malloc(slv->options.levels*sizeof(ef_level));
// setup callback for transforming rhs on coarse levels
} else {
slv->levels = (ef_level*) malloc(sizeof(ef_level));
}
ierr = ef_fd_create(&slv->fd, EF_FD_STANDARD_O2);CHKERRQ(ierr);
ierr = ef_operator_create(&slv->op, slv->levels, slv->fd, slv->grid.nd);CHKERRQ(ierr);
ierr = ef_boundary_create(&slv->boundary, slv->levels, slv->options.levels,
slv->dmap, &slv->state, slv->fd);CHKERRQ(ierr);
slv->op->axisymmetric = slv->options.axisymmetric;
slv->boundary->axisymmetric = slv->options.axisymmetric;
ierr = DMSetMatType(slv->dm, MATAIJ);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].eps);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].g);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].ag);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].gcomp);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].scale);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(slv->dm, &slv->levels[0].nscale);CHKERRQ(ierr);
ierr = VecSet(slv->levels[0].g, 0);CHKERRQ(ierr);
ierr = VecSet(slv->levels[0].gcomp, 1);CHKERRQ(ierr);
ierr = VecSet(slv->levels[0].scale, 1);CHKERRQ(ierr);
ierr = VecSet(slv->levels[0].nscale, 1);CHKERRQ(ierr);
ierr = DMCoarsenHookAdd(slv->dm, slv->callback->coarsen, slv->callback->restrct, slv);CHKERRQ(ierr);
return 0;
}
bool PETScLinearSolver::solve(PETScMatrix& A, PETScVector& b, PETScVector& x)
{
BaseLib::RunTime wtimer;
wtimer.start();
// define TEST_MEM_PETSC
#ifdef TEST_MEM_PETSC
PetscLogDouble mem1, mem2;
PetscMemoryGetCurrentUsage(&mem1);
#endif
#if (PETSC_VERSION_NUMBER > 3040)
KSPSetOperators(_solver, A.getRawMatrix(), A.getRawMatrix());
#else
KSPSetOperators(_solver, A.getRawMatrix(), A.getRawMatrix(),
DIFFERENT_NONZERO_PATTERN);
#endif
KSPSolve(_solver, b.getRawVector(), x.getRawVector());
KSPConvergedReason reason;
KSPGetConvergedReason(_solver, &reason);
bool converged = true;
if (reason > 0)
{
const char* ksp_type;
const char* pc_type;
KSPGetType(_solver, &ksp_type);
PCGetType(_pc, &pc_type);
PetscPrintf(PETSC_COMM_WORLD,
"\n================================================");
PetscPrintf(PETSC_COMM_WORLD,
"\nLinear solver %s with %s preconditioner", ksp_type,
pc_type);
PetscInt its;
KSPGetIterationNumber(_solver, &its);
PetscPrintf(PETSC_COMM_WORLD, "\nconverged in %d iterations", its);
switch (reason)
{
case KSP_CONVERGED_RTOL:
PetscPrintf(PETSC_COMM_WORLD,
" (relative convergence criterion fulfilled).");
break;
case KSP_CONVERGED_ATOL:
PetscPrintf(PETSC_COMM_WORLD,
" (absolute convergence criterion fulfilled).");
break;
default:
PetscPrintf(PETSC_COMM_WORLD, ".");
}
PetscPrintf(PETSC_COMM_WORLD,
"\n================================================\n");
}
else if (reason == KSP_DIVERGED_ITS)
{
const char* ksp_type;
const char* pc_type;
KSPGetType(_solver, &ksp_type);
PCGetType(_pc, &pc_type);
PetscPrintf(PETSC_COMM_WORLD,
"\nLinear solver %s with %s preconditioner", ksp_type,
pc_type);
PetscPrintf(PETSC_COMM_WORLD,
"\nWarning: maximum number of iterations reached.\n");
}
else
{
converged = false;
if (reason == KSP_DIVERGED_INDEFINITE_PC)
{
PetscPrintf(PETSC_COMM_WORLD,
"\nDivergence because of indefinite preconditioner,");
PetscPrintf(PETSC_COMM_WORLD,
"\nTry to run again with "
"-pc_factor_shift_positive_definite option.\n");
}
else if (reason == KSP_DIVERGED_BREAKDOWN_BICG)
{
PetscPrintf(PETSC_COMM_WORLD,
"\nKSPBICG method was detected so the method could not "
"continue to enlarge the Krylov space.");
PetscPrintf(PETSC_COMM_WORLD,
"\nTry to run again with another solver.\n");
}
else if (reason == KSP_DIVERGED_NONSYMMETRIC)
{
PetscPrintf(PETSC_COMM_WORLD,
"\nMatrix or preconditioner is unsymmetric but KSP "
"requires symmetric.\n");
}
else
{
PetscPrintf(PETSC_COMM_WORLD,
"\nDivergence detected, use command option "
"-ksp_monitor or -log_summary to check the details.\n");
}
}
#ifdef TEST_MEM_PETSC
PetscMemoryGetCurrentUsage(&mem2);
PetscPrintf(
PETSC_COMM_WORLD,
"###Memory usage by solver. Before: %f After: %f Increase: %d\n", mem1,
mem2, (int)(mem2 - mem1));
#endif
_elapsed_ctime += wtimer.elapsed();
return converged;
}
PetscErrorCode TSSetUp_Sundials(TS ts)
{
TS_Sundials *cvode = (TS_Sundials*)ts->data;
PetscErrorCode ierr;
PetscInt glosize,locsize,i,flag;
PetscScalar *y_data,*parray;
void *mem;
PC pc;
PCType pctype;
PetscBool pcnone;
PetscFunctionBegin;
/* get the vector size */
ierr = VecGetSize(ts->vec_sol,&glosize);CHKERRQ(ierr);
ierr = VecGetLocalSize(ts->vec_sol,&locsize);CHKERRQ(ierr);
/* allocate the memory for N_Vec y */
cvode->y = N_VNew_Parallel(cvode->comm_sundials,locsize,glosize);
if (!cvode->y) SETERRQ(PETSC_COMM_SELF,1,"cvode->y is not allocated");
/* initialize N_Vec y: copy ts->vec_sol to cvode->y */
ierr = VecGetArray(ts->vec_sol,&parray);CHKERRQ(ierr);
y_data = (PetscScalar*) N_VGetArrayPointer(cvode->y);
for (i = 0; i < locsize; i++) y_data[i] = parray[i];
ierr = VecRestoreArray(ts->vec_sol,NULL);CHKERRQ(ierr);
ierr = VecDuplicate(ts->vec_sol,&cvode->update);CHKERRQ(ierr);
ierr = VecDuplicate(ts->vec_sol,&cvode->ydot);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)ts,(PetscObject)cvode->update);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)ts,(PetscObject)cvode->ydot);CHKERRQ(ierr);
/*
Create work vectors for the TSPSolve_Sundials() routine. Note these are
allocated with zero space arrays because the actual array space is provided
by Sundials and set using VecPlaceArray().
*/
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)ts),1,locsize,PETSC_DECIDE,0,&cvode->w1);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)ts),1,locsize,PETSC_DECIDE,0,&cvode->w2);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)ts,(PetscObject)cvode->w1);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)ts,(PetscObject)cvode->w2);CHKERRQ(ierr);
/* Call CVodeCreate to create the solver memory and the use of a Newton iteration */
mem = CVodeCreate(cvode->cvode_type, CV_NEWTON);
if (!mem) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"CVodeCreate() fails");
cvode->mem = mem;
/* Set the pointer to user-defined data */
flag = CVodeSetUserData(mem, ts);
if (flag) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"CVodeSetUserData() fails");
/* Sundials may choose to use a smaller initial step, but will never use a larger step. */
flag = CVodeSetInitStep(mem,(realtype)ts->time_step);
if (flag) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_LIB,"CVodeSetInitStep() failed");
if (cvode->mindt > 0) {
flag = CVodeSetMinStep(mem,(realtype)cvode->mindt);
if (flag) {
if (flag == CV_MEM_NULL) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_LIB,"CVodeSetMinStep() failed, cvode_mem pointer is NULL");
else if (flag == CV_ILL_INPUT) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_LIB,"CVodeSetMinStep() failed, hmin is nonpositive or it exceeds the maximum allowable step size");
else SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_LIB,"CVodeSetMinStep() failed");
}
}
if (cvode->maxdt > 0) {
flag = CVodeSetMaxStep(mem,(realtype)cvode->maxdt);
if (flag) SETERRQ(PetscObjectComm((PetscObject)ts),PETSC_ERR_LIB,"CVodeSetMaxStep() failed");
}
/* Call CVodeInit to initialize the integrator memory and specify the
* user's right hand side function in u'=f(t,u), the inital time T0, and
* the initial dependent variable vector cvode->y */
flag = CVodeInit(mem,TSFunction_Sundials,ts->ptime,cvode->y);
if (flag) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CVodeInit() fails, flag %d",flag);
/* specifies scalar relative and absolute tolerances */
flag = CVodeSStolerances(mem,cvode->reltol,cvode->abstol);
if (flag) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CVodeSStolerances() fails, flag %d",flag);
/* Specify max num of steps to be taken by cvode in its attempt to reach the next output time */
flag = CVodeSetMaxNumSteps(mem,ts->max_steps);
/* call CVSpgmr to use GMRES as the linear solver. */
/* setup the ode integrator with the given preconditioner */
ierr = TSSundialsGetPC(ts,&pc);CHKERRQ(ierr);
ierr = PCGetType(pc,&pctype);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)pc,PCNONE,&pcnone);CHKERRQ(ierr);
if (pcnone) {
flag = CVSpgmr(mem,PREC_NONE,0);
if (flag) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CVSpgmr() fails, flag %d",flag);
} else {
flag = CVSpgmr(mem,PREC_LEFT,cvode->maxl);
if (flag) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CVSpgmr() fails, flag %d",flag);
/* Set preconditioner and solve routines Precond and PSolve,
and the pointer to the user-defined block data */
flag = CVSpilsSetPreconditioner(mem,TSPrecond_Sundials,TSPSolve_Sundials);
if (flag) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CVSpilsSetPreconditioner() fails, flag %d", flag);
}
flag = CVSpilsSetGSType(mem, MODIFIED_GS);
if (flag) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CVSpgmrSetGSType() fails, flag %d",flag);
PetscFunctionReturn(0);
}
void TestBasicFunctionality() throw (Exception)
{
/*
* We need to make sure here that the matrix is loaded with the appropriate parallel layout. Petsc's
* default puts 1331 rows in each processor. This wouldn't be possible in a real bidomain simulation
* because implies that equations V_665 an Phi_e_665 are solved in different processors.
*/
unsigned num_nodes = 1331;
DistributedVectorFactory factory(num_nodes);
Vec parallel_layout = factory.CreateVec(2);
Mat system_matrix;
PetscTools::ReadPetscObject(system_matrix, "linalg/test/data/matrices/cube_6000elems_half_activated.mat", parallel_layout);
PetscTools::Destroy(parallel_layout);
// Set rhs = A * [1 0 1 0 ... 1 0]'
Vec one_zeros = factory.CreateVec(2);
Vec rhs = factory.CreateVec(2);
for (unsigned node_index=0; node_index<2*num_nodes; node_index+=2)
{
PetscVecTools::SetElement(one_zeros, node_index, 1.0);
PetscVecTools::SetElement(one_zeros, node_index+1, 0.0);
}
PetscVecTools::Finalise(one_zeros);
MatMult(system_matrix, one_zeros, rhs);
PetscTools::Destroy(one_zeros);
LinearSystem ls = LinearSystem(rhs, system_matrix);
ls.SetAbsoluteTolerance(1e-9);
ls.SetKspType("cg");
ls.SetPcType("none");
ls.AssembleFinalLinearSystem();
Vec solution = ls.Solve();
DistributedVector distributed_solution = factory.CreateDistributedVector(solution);
DistributedVector::Stripe vm(distributed_solution, 0);
DistributedVector::Stripe phi_e(distributed_solution, 1);
for (DistributedVector::Iterator index = distributed_solution.Begin();
index!= distributed_solution.End();
++index)
{
/*
* Although we're trying to enforce the solution to be [1 0 ... 1 0], the system is singular and
* therefore it has infinite solutions. I've (migb) found that the use of different preconditioners
* lead to different solutions ([0.8 -0.2 ... 0.8 -0.2], [0.5 -0.5 ... 0.5 -0.5], ...)
*
* If we were using PETSc null space, it would find the solution that satisfies x'*v=0,
* being v the null space of the system (v=[1 1 ... 1])
*/
TS_ASSERT_DELTA(vm[index] - phi_e[index], 1.0, 1e-6);
}
// Coverage (setting PC type after first solve)
ls.SetPcType("blockdiagonal");
PetscTools::Destroy(system_matrix);
PetscTools::Destroy(rhs);
PetscTools::Destroy(solution);
#if (PETSC_VERSION_MAJOR == 3 && PETSC_VERSION_MINOR <= 3) //PETSc 3.0 to PETSc 3.3
//The PETSc developers changed this one, but later changed it back again!
const PCType pc;
#else
PCType pc;
#endif
PC prec;
KSPGetPC(ls.mKspSolver, &prec);
PCGetType(prec, &pc);
// Although we call it "blockdiagonal", PETSc considers this PC a generic SHELL preconditioner
TS_ASSERT( strcmp(pc,"shell")==0 );
}