int main(int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start %s\n", __FILE__); CHKERRQ(ierr);
MPI_Comm comm = PETSC_COMM_WORLD;
DA da;
Mat mat;
int MX = 100;
int dof = 3;
ierr = PetscPrintf(comm,"Started Assembly\n"); CHKERRQ(ierr);
ierr = MatMake(MX,dof,&da, &mat); CHKERRQ(ierr);
ierr = MatWrite("J",mat); CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Finished Assembly\n"); CHKERRQ(ierr);
Vec rhs, sol;
ierr = DACreateGlobalVector(da,&rhs); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&sol); CHKERRQ(ierr);
int i,j;
int xs,ys,xm,ym;
Field **array;
ierr = DAVecGetArray(da,rhs,&array); CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL); CHKERRQ(ierr);
for (j = xs; j < xm; ++j) {
for (i = ys; i < ym; ++i) {
if( 3*MX/4 > i && i > MX/4 &&
3*MX/4 > j && j > MX/4 )
{
array[j][i].u = 100.;
array[j][i].v = 100.;
}
}
}
ierr = DAVecRestoreArray(da,rhs,&array); CHKERRQ(ierr);
KSP ksp;
ierr = KSPCreate(comm,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY); CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,mat,mat,DIFFERENT_NONZERO_PATTERN); CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
//Split pressure from velocity
PC pc;
ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr);
ierr = PCFieldSplitSetFields(pc,2,(int[]){0,1}); CHKERRQ(ierr);
PETSC_EXTERN void PETSC_STDCALL pcfieldsplitsettype_(PC pc,PCCompositeType *type, int *__ierr ){
*__ierr = PCFieldSplitSetType(
(PC)PetscToPointer((pc) ),*type);
}
PetscErrorCode FluidFieldSetup( FluidField f )
{
PetscLogDouble t1,t2;
PetscErrorCode ierr;
PetscFunctionBegin;
// Assemble viscous matricies
ierr = FluidFieldMatAssemble( f ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Lengths: %e %e %e\n", f->lens.x, f->lens.y, f->lens.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Size: %d %d %d\n", f->dims.x, f->dims.y, f->dims.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "dx: %e %e %e\n", f->dh.x, f->dh.y, f->dh.z ); CHKERRQ(ierr);
ierr = PetscTime(&t1); CHKERRQ(ierr);
// Create vectors
ierr = GACreate( f->daV, &f->ga); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daV,&f->rhs); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel0); CHKERRQ(ierr);
// ierr = DACreateGlobalVector(f->daE,&f->E); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daB,&f->buf); CHKERRQ(ierr);
// Set up the outer solver
ierr = KSPCreate(f->comm,&f->ksp); CHKERRQ(ierr);
ierr = KSPSetOperators(f->ksp,f->mat,f->mat, SAME_PRECONDITIONER); CHKERRQ(ierr);
ierr = KSPSetType(f->ksp,KSPFGMRES); CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(f->ksp,PETSC_TRUE); CHKERRQ(ierr);
ierr = KSPSetTolerances(f->ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr);
ierr = KSPSetFromOptions(f->ksp);CHKERRQ(ierr);
// Split pressure from velocity [ u v w | p ]
PC pc;
ierr = KSPGetPC(f->ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
if( f->is3D ) {
const PetscInt ufields[] = {U_FACE,V_FACE,W_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,4); CHKERRQ(ierr); // [p u v w]
ierr = PCFieldSplitSetFields(pc,"v",3,ufields,ufields); CHKERRQ(ierr); // [u v w]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
} else {
const PetscInt ufields[] = {U_FACE,V_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr); // [p u v]
ierr = PCFieldSplitSetFields(pc,"v",2,ufields,ufields); CHKERRQ(ierr); // [u v]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
}
ierr = PCSetUp(pc); CHKERRQ(ierr);
int nVelP;
KSP *kspVelP;
ierr = PCFieldSplitGetSubKSP(pc,&nVelP,&kspVelP); CHKERRQ(ierr);
ierr = KSPSetTolerances(kspVelP[1],PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,4); CHKERRQ(ierr);
ierr = KSPSetType(kspVelP[1],KSPGMRES); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[1],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspVelP[1]);CHKERRQ(ierr);
// Split velocity [u v w] into component matricies [u], [v], [w]
ierr = KSPSetType(kspVelP[0],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[0],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_ADDITIVE); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,f->is3D?3:2); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
/* Set solver for each velocity component
* Split component velocity as parallel blocks along processors
* Use direct solver for each block
* TODO: use MG, w/FFT on coarse grid
*/
ierr = PetscTime(&t2); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Finished Solver Setup: %f sec\n",t2-t1); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void
SolverLinearPetsc<T>::setPetscPreconditionerType()
{
int ierr = 0;
#if (PETSC_VERSION_MAJOR == 3) && (PETSC_VERSION_MINOR >= 2)
ierr = PCFactorSetMatSolverPackage( M_pc,MATSOLVERUMFPACK );
if ( ierr )
{
ierr = PCFactorSetMatSolverPackage( M_pc,MATSOLVERSUPERLU );
if ( ierr )
{
ierr = PCFactorSetMatSolverPackage( M_pc,MATSOLVERPETSC );
}
}
#elif (PETSC_VERSION_MAJOR >= 3)
ierr = PCFactorSetMatSolverPackage( M_pc,MAT_SOLVER_UMFPACK );
if ( ierr )
{
ierr = PCFactorSetMatSolverPackage( M_pc,MAT_SOLVER_SUPERLU );
if ( ierr )
{
ierr = PCFactorSetMatSolverPackage( M_pc,MAT_SOLVER_PETSC );
}
}
#endif
DVLOG(2) << "[SolverLinearPetsc] preconditioner type: " << this->preconditionerType() << "\n";
switch ( this->preconditionerType() )
{
case IDENTITY_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCNONE );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case CHOLESKY_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCCHOLESKY );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case ICC_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCICC );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case ILU_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCILU );
CHKERRABORT( this->worldComm().globalComm(),ierr );
if ( this->vm().count( "pc-factor-levels" ) )
{
PCFactorSetLevels( M_pc,this->vm()["pc-factor-levels"].template as<int>() );
}
else
PCFactorSetLevels( M_pc,3 );
if ( this->vm().count( "pc-factor-fill" ) )
{
PCFactorSetFill( M_pc,this->vm()["pc-factor-fill"].template as<double>() );
}
else
PCFactorSetFill( M_pc,40 );
return;
case LU_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCLU );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case ASM_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCASM );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
#if PETSC_VERSION_GREATER_OR_EQUAL_THAN( 3,2,0 )
case GASM_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCGASM );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
#endif
case JACOBI_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCJACOBI );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case BLOCK_JACOBI_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCBJACOBI );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case SOR_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCSOR );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case EISENSTAT_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCEISENSTAT );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
#if !((PETSC_VERSION_MAJOR == 2) && (PETSC_VERSION_MINOR <= 1) && (PETSC_VERSION_SUBMINOR <= 1))
case USER_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCMAT );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
#endif
case SHELL_PRECOND:
ierr = PCSetType ( M_pc, ( char* ) PCSHELL );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case FIELDSPLIT_PRECOND:
ierr = PCSetType( M_pc,( char* ) PCFIELDSPLIT );
CHKERRABORT( this->worldComm().globalComm(),ierr );
ierr = PCFieldSplitSetType( M_pc,PC_COMPOSITE_SCHUR );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
case ML_PRECOND:
ierr = PCSetType( M_pc,( char* ) PCML );
CHKERRABORT( this->worldComm().globalComm(),ierr );
return;
default:
std::cerr << "ERROR: Unsupported PETSC Preconditioner: "
<< this->preconditionerType() << std::endl
<< "Continuing with PETSC defaults" << std::endl;
}
}
ierr = PCFieldSplitSetFields(pc,2,(int[]){0,1}); CHKERRQ(ierr);
ierr = PCFieldSplitSetFields(pc,1,(int[]){2}); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
//Split velocity into u-v component matricies
int nVelP;
KSP *kspVelP;
ierr = PCFieldSplitGetSubKSP(pc,&nVelP,&kspVelP); CHKERRQ(ierr);
ierr = KSPSetTolerances(kspVelP[1],1e-50,1e-3,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr);
// ierr = KSPSetType(kspVelP[1],KSPCG); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[1],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
ierr = KSPSetType(kspVelP[0],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[0],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_ADDITIVE); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,2); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
//Set solver for each velocity component
int nVel;
KSP *kspVel;
ierr = PCFieldSplitGetSubKSP(pc,&nVel,&kspVel); CHKERRQ(ierr);
for( i = 0; i < nVel; i++ ) {
// Direct Method
ierr = KSPSetType(kspVel[i],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVel[i],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCCHOLESKY); CHKERRQ(ierr);
ierr = PCFactorSetMatOrderingType(pc,MATORDERING_ND); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
//