PetscErrorCode StokesSetupPC(Stokes *s, KSP ksp)
{
KSP *subksp;
PC pc, pc1;
PetscInt n = 1;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc, "0", s->isg[0]);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc, "1", s->isg[1]);CHKERRQ(ierr);
if (s->userPC) {
ierr = PCFieldSplitSetSchurPre(pc, PC_FIELDSPLIT_SCHUR_PRE_USER, s->myS);CHKERRQ(ierr);
}
if (s->userKSP) {
ierr = PCSetUp(pc);CHKERRQ(ierr);
ierr = PCFieldSplitGetSubKSP(pc, &n, &subksp);CHKERRQ(ierr);
ierr = KSPSetOperators(subksp[1], s->myS, s->myS);CHKERRQ(ierr);
ierr = KSPGetPC(subksp[1], &pc1);CHKERRQ(ierr);
ierr = PCSetUp(pc1);CHKERRQ(ierr);
ierr = PCSetType(pc1,PCGAMG);CHKERRQ(ierr);
ierr = PetscFree(subksp);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
PC pc;
PetscErrorCode ierr;
PetscInt n = 5;
Mat mat;
ierr = PetscInitialize(&argc,&args,(char*)0,help);
if (ierr) return ierr;
ierr = PCCreate(PETSC_COMM_WORLD,&pc);
CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE);
CHKERRQ(ierr);
/* Vector and matrix must be set before calling PCSetUp */
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,n,n,3,NULL,&mat);
CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat);
CHKERRQ(ierr);
ierr = PCSetUp(pc);
CHKERRQ(ierr);
ierr = MatDestroy(&mat);
CHKERRQ(ierr);
ierr = PCDestroy(&pc);
CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat mat;
Vec b,u;
PC pc;
PetscErrorCode ierr;
PetscInt n = 5,i,col[3];
PetscScalar value[3];
PetscInitialize(&argc,&args,(char *)0,help);
/* Create vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&b);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&u);CHKERRQ(ierr);
/* Create and assemble matrix */
ierr = MatCreateSeqDense(PETSC_COMM_SELF,n,n,PETSC_NULL,&mat);CHKERRQ(ierr);
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(mat,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0] = n - 2; col[1] = n - 1;
ierr = MatSetValues(mat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0; col[0] = 0; col[1] = 1; value[0] = 2.0; value[1] = -1.0;
ierr = MatSetValues(mat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Create PC context and set up data structures */
ierr = PCCreate(PETSC_COMM_WORLD,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCSOR);CHKERRQ(ierr);
ierr = PCSetFromOptions(pc);CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = PCSetUp(pc);CHKERRQ(ierr);
value[0] = 1.0;
for (i=0; i<n; i++) {
ierr = VecSet(u,0.0);CHKERRQ(ierr);
ierr = VecSetValues(u,1,&i,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(u);CHKERRQ(ierr);
ierr = VecAssemblyEnd(u);CHKERRQ(ierr);
ierr = PCApply(pc,u,b);CHKERRQ(ierr);
ierr = VecView(b,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
/* Free data structures */
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = PCDestroy(&pc);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void PetscPreconditioner<T>::set_petsc_subpreconditioner_type(const PCType type, PC& pc)
#endif
{
// For catching PETSc error return codes
int ierr = 0;
// get the communicator from the PETSc object
Parallel::communicator comm;
PetscObjectGetComm((PetscObject)pc, &comm);
Parallel::Communicator communicator(comm);
// All docs say must call KSPSetUp or PCSetUp before calling PCBJacobiGetSubKSP.
// You must call PCSetUp after the preconditioner operators have been set, otherwise you get the:
//
// "Object is in wrong state!"
// "Matrix must be set first."
//
// error messages...
ierr = PCSetUp(pc);
CHKERRABORT(comm,ierr);
// To store array of local KSP contexts on this processor
KSP* subksps;
// the number of blocks on this processor
PetscInt n_local;
// The global number of the first block on this processor.
// This is not used, so we just pass PETSC_NULL instead.
// int first_local;
// Fill array of local KSP contexts
ierr = PCBJacobiGetSubKSP(pc, &n_local, PETSC_NULL, &subksps);
CHKERRABORT(comm,ierr);
// Loop over sub-ksp objects, set ILU preconditioner
for (PetscInt i=0; i<n_local; ++i)
{
// Get pointer to sub KSP object's PC
PC subpc;
ierr = KSPGetPC(subksps[i], &subpc);
CHKERRABORT(comm,ierr);
// Set requested type on the sub PC
ierr = PCSetType(subpc, type);
CHKERRABORT(comm,ierr);
}
}
int main(int argc,char **args)
{
Mat C,A;
PetscInt i,j;
PetscErrorCode ierr;
PetscScalar v;
PC pc;
Vec xtmp;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,3,3);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_WORLD,3,&xtmp);CHKERRQ(ierr);
i = 0; j = 0; v = 4;
ierr = MatSetValues(C,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
i = 0; j = 2; v = 1;
ierr = MatSetValues(C,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
i = 1; j = 0; v = 1;
ierr = MatSetValues(C,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
i = 1; j = 1; v = 4;
ierr = MatSetValues(C,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
i = 2; j = 1; v = 1;
ierr = MatSetValues(C,1,&i,1,&j,&v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PCCreate(PETSC_COMM_WORLD,&pc);CHKERRQ(ierr);
ierr = PCSetFromOptions(pc);CHKERRQ(ierr);
ierr = PCSetOperators(pc,C,C,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = PCSetUp(pc);CHKERRQ(ierr);
ierr = PCFactorGetMatrix(pc,&A);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PCDestroy(&pc);CHKERRQ(ierr);
ierr = VecDestroy(&xtmp);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
PC pc;
PetscErrorCode ierr;
PetscInt n = 5;
Mat mat;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = PCCreate(PETSC_COMM_WORLD,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE);CHKERRQ(ierr);
/* Vector and matrix must be set before calling PCSetUp */
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,n,n,3,PETSC_NULL,&mat);CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = PCSetUp(pc);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = PCDestroy(&pc); CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void setCompositePC(Type* A, PC pc) {
PetscInt nsplits;
KSP* subksp;
PCFieldSplitGetSubKSP(pc, &nsplits, &subksp);
if(A->_S.size() == 1)
KSPSetOperators(subksp[nsplits - 1], A->_S[0], A->_S[0]);
else {
PC pcS;
KSPGetPC(subksp[nsplits - 1], &pcS);
for(int i = 0; i < A->_S.size(); ++i)
PCCompositeAddPC(pcS, PCNONE);
PCSetUp(pcS);
for(int i = 0; i < A->_S.size(); ++i) {
PC subpc;
PCCompositeGetPC(pcS, i, &subpc);
PCSetOperators(subpc, A->_S[i], A->_S[i]);
PCSetFromOptions(subpc);
}
}
PetscFree(subksp);
}
PetscErrorCode PressurePoissonCreate( )
{
PetscErrorCode ierr;
PetscFunctionBegin;
PetscLogEventBegin(EVENT_PressurePoissonCreate,0,0,0,0);
// PetscLogEventRegister(&EVENT_PressurePoissonCreate,"PressurePoissonCreate", 0);
ierr = KSPCreate(PETSC_COMM_SELF, &ksp); CHKERRQ(ierr);
KSPGetPC(ksp, &pc);
// KSPSetType(ksp, KSPCG);
// PCSetType(pc, PCICC);
// PCFactorSetLevels(pc, 0);
// KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
KSPSetType(ksp, KSPPREONLY);
PCSetType(pc, PCCHOLESKY);
PCFactorSetMatOrderingType(pc, MATORDERING_ND);
KSPSetFromOptions(ksp);
KSPSetOperators(ksp, m, m, SAME_PRECONDITIONER);
PCSetUp(pc);
ierr = IIMCreate(&iim, 12); CHKERRQ(ierr);
// IIMSetForceComponents(iim, ForceComponentNormalSimple, ForceComponentTangentialSimple);
CreateGrid2D(d1, d2, &rhsC);
CreateGrid2D(d1, d2, &p);
CreateGrid2D(d1, d2, &px);
CreateGrid2D(d1, d2, &py);
CreateGrid2D(d1, d2, &u);
CreateGrid2D(d1, d2, &v);
ierr = CreateLevelSet2D(d1,d2,&ls); CHKERRQ(ierr);
CreateLevelSet2D(d1, d2, &lstemp);
LevelSetInitializeToStar(ls,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE);
// LevelSetInitializeToCircle(ls, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE);
PetscLogEventEnd(EVENT_PressurePoissonCreate,0,0,0,0);
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCNullSpaceAssembleCorrection(PC pc, PetscBool isdir, IS local_dofs)
{
PC_BDDC *pcbddc = (PC_BDDC*)pc->data;
PC_IS *pcis = (PC_IS*)pc->data;
Mat_IS* matis = (Mat_IS*)pc->pmat->data;
KSP local_ksp;
PC newpc;
NullSpaceCorrection_ctx shell_ctx;
Mat local_mat,local_pmat,small_mat,inv_small_mat;
Vec work1,work2;
const Vec *nullvecs;
VecScatter scatter_ctx;
IS is_aux;
MatFactorInfo matinfo;
PetscScalar *basis_mat,*Kbasis_mat,*array,*array_mat;
PetscScalar one = 1.0,zero = 0.0, m_one = -1.0;
PetscInt basis_dofs,basis_size,nnsp_size,i,k;
PetscBool nnsp_has_cnst;
PetscErrorCode ierr;
PetscFunctionBegin;
/* Infer the local solver */
ierr = ISGetSize(local_dofs,&basis_dofs);CHKERRQ(ierr);
if (isdir) {
/* Dirichlet solver */
local_ksp = pcbddc->ksp_D;
} else {
/* Neumann solver */
local_ksp = pcbddc->ksp_R;
}
ierr = KSPGetOperators(local_ksp,&local_mat,&local_pmat);CHKERRQ(ierr);
/* Get null space vecs */
ierr = MatNullSpaceGetVecs(pcbddc->NullSpace,&nnsp_has_cnst,&nnsp_size,&nullvecs);CHKERRQ(ierr);
basis_size = nnsp_size;
if (nnsp_has_cnst) {
basis_size++;
}
if (basis_dofs) {
/* Create shell ctx */
ierr = PetscNew(&shell_ctx);CHKERRQ(ierr);
/* Create work vectors in shell context */
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_small_1,basis_size,basis_size);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_small_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_small_1,&shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&shell_ctx->work_full_1);CHKERRQ(ierr);
ierr = VecSetSizes(shell_ctx->work_full_1,basis_dofs,basis_dofs);CHKERRQ(ierr);
ierr = VecSetType(shell_ctx->work_full_1,VECSEQ);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&shell_ctx->work_full_2);CHKERRQ(ierr);
/* Allocate workspace */
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->basis_mat );CHKERRQ(ierr);
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_dofs,basis_size,NULL,&shell_ctx->Kbasis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Restrict local null space on selected dofs (Dirichlet or Neumann)
and compute matrices N and K*N */
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,local_dofs,work1,(IS)0,&scatter_ctx);CHKERRQ(ierr);
}
for (k=0;k<nnsp_size;k++) {
ierr = VecScatterBegin(matis->rctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->rctx,nullvecs[k],pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (basis_dofs) {
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter_ctx,pcis->vec1_N,work1,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
}
if (basis_dofs) {
if (nnsp_has_cnst) {
ierr = VecPlaceArray(work1,(const PetscScalar*)&basis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = VecSet(work1,one);CHKERRQ(ierr);
ierr = VecPlaceArray(work2,(const PetscScalar*)&Kbasis_mat[k*basis_dofs]);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = VecResetArray(work1);CHKERRQ(ierr);
ierr = VecResetArray(work2);CHKERRQ(ierr);
}
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter_ctx);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->basis_mat,&basis_mat);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(shell_ctx->Kbasis_mat,&Kbasis_mat);CHKERRQ(ierr);
/* Assemble another Mat object in shell context */
ierr = MatTransposeMatMult(shell_ctx->basis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&small_mat);CHKERRQ(ierr);
ierr = MatFactorInfoInitialize(&matinfo);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,basis_size,0,1,&is_aux);CHKERRQ(ierr);
ierr = MatLUFactor(small_mat,is_aux,is_aux,&matinfo);CHKERRQ(ierr);
ierr = ISDestroy(&is_aux);CHKERRQ(ierr);
ierr = PetscMalloc1(basis_size*basis_size,&array_mat);CHKERRQ(ierr);
for (k=0;k<basis_size;k++) {
ierr = VecSet(shell_ctx->work_small_1,zero);CHKERRQ(ierr);
ierr = VecSetValue(shell_ctx->work_small_1,k,one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = VecAssemblyEnd(shell_ctx->work_small_1);CHKERRQ(ierr);
ierr = MatSolve(small_mat,shell_ctx->work_small_1,shell_ctx->work_small_2);CHKERRQ(ierr);
ierr = VecGetArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
for (i=0;i<basis_size;i++) {
array_mat[i*basis_size+k]=array[i];
}
ierr = VecRestoreArrayRead(shell_ctx->work_small_2,(const PetscScalar**)&array);CHKERRQ(ierr);
}
ierr = MatCreateSeqDense(PETSC_COMM_SELF,basis_size,basis_size,array_mat,&inv_small_mat);CHKERRQ(ierr);
ierr = MatMatMult(shell_ctx->basis_mat,inv_small_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&shell_ctx->Lbasis_mat);CHKERRQ(ierr);
ierr = PetscFree(array_mat);CHKERRQ(ierr);
ierr = MatDestroy(&inv_small_mat);CHKERRQ(ierr);
ierr = MatDestroy(&small_mat);CHKERRQ(ierr);
ierr = MatScale(shell_ctx->Kbasis_mat,m_one);CHKERRQ(ierr);
/* Rebuild local PC */
ierr = KSPGetPC(local_ksp,&shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)shell_ctx->local_pc);CHKERRQ(ierr);
ierr = PCCreate(PETSC_COMM_SELF,&newpc);CHKERRQ(ierr);
ierr = PCSetOperators(newpc,local_mat,local_mat);CHKERRQ(ierr);
ierr = PCSetType(newpc,PCSHELL);CHKERRQ(ierr);
ierr = PCShellSetContext(newpc,shell_ctx);CHKERRQ(ierr);
ierr = PCShellSetApply(newpc,PCBDDCApplyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCShellSetDestroy(newpc,PCBDDCDestroyNullSpaceCorrectionPC);CHKERRQ(ierr);
ierr = PCSetUp(newpc);CHKERRQ(ierr);
ierr = KSPSetPC(local_ksp,newpc);CHKERRQ(ierr);
ierr = PCDestroy(&newpc);CHKERRQ(ierr);
ierr = KSPSetUp(local_ksp);CHKERRQ(ierr);
}
/* test */
if (pcbddc->dbg_flag && basis_dofs) {
KSP check_ksp;
PC check_pc;
Mat test_mat;
Vec work3;
PetscReal test_err,lambda_min,lambda_max;
PetscBool setsym,issym=PETSC_FALSE;
PetscInt tabs;
ierr = PetscViewerASCIIGetTab(pcbddc->dbg_viewer,&tabs);CHKERRQ(ierr);
ierr = KSPGetPC(local_ksp,&check_pc);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work1);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work2);CHKERRQ(ierr);
ierr = VecDuplicate(shell_ctx->work_full_1,&work3);CHKERRQ(ierr);
ierr = VecSetRandom(shell_ctx->work_small_1,NULL);CHKERRQ(ierr);
ierr = MatMult(shell_ctx->basis_mat,shell_ctx->work_small_1,work1);CHKERRQ(ierr);
ierr = VecCopy(work1,work2);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work3);CHKERRQ(ierr);
ierr = PCApply(check_pc,work3,work1);CHKERRQ(ierr);
ierr = VecAXPY(work1,m_one,work2);CHKERRQ(ierr);
ierr = VecNorm(work1,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for nullspace correction for ",PetscGlobalRank);CHKERRQ(ierr);
ierr = PetscViewerASCIIUseTabs(pcbddc->dbg_viewer,PETSC_FALSE);CHKERRQ(ierr);
if (isdir) {
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Dirichlet ");CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Neumann ");CHKERRQ(ierr);
}
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"solver is :%1.14e\n",test_err);CHKERRQ(ierr);
ierr = PetscViewerASCIISetTab(pcbddc->dbg_viewer,tabs);CHKERRQ(ierr);
ierr = PetscViewerASCIIUseTabs(pcbddc->dbg_viewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatTransposeMatMult(shell_ctx->Lbasis_mat,shell_ctx->Kbasis_mat,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&test_mat);CHKERRQ(ierr);
ierr = MatShift(test_mat,one);CHKERRQ(ierr);
ierr = MatNorm(test_mat,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = MatDestroy(&test_mat);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for nullspace matrices is :%1.14e\n",PetscGlobalRank,test_err);CHKERRQ(ierr);
/* Create ksp object suitable for extreme eigenvalues' estimation */
ierr = KSPCreate(PETSC_COMM_SELF,&check_ksp);CHKERRQ(ierr);
ierr = KSPSetErrorIfNotConverged(check_ksp,pc->erroriffailure);CHKERRQ(ierr);
ierr = KSPSetOperators(check_ksp,local_mat,local_mat);CHKERRQ(ierr);
ierr = KSPSetTolerances(check_ksp,1.e-8,1.e-8,PETSC_DEFAULT,basis_dofs);CHKERRQ(ierr);
ierr = KSPSetComputeSingularValues(check_ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatIsSymmetricKnown(pc->pmat,&setsym,&issym);CHKERRQ(ierr);
if (issym) {
ierr = KSPSetType(check_ksp,KSPCG);CHKERRQ(ierr);
}
ierr = KSPSetPC(check_ksp,check_pc);CHKERRQ(ierr);
ierr = KSPSetUp(check_ksp);CHKERRQ(ierr);
ierr = VecSetRandom(work1,NULL);CHKERRQ(ierr);
ierr = MatMult(local_mat,work1,work2);CHKERRQ(ierr);
ierr = KSPSolve(check_ksp,work2,work2);CHKERRQ(ierr);
ierr = VecAXPY(work2,m_one,work1);CHKERRQ(ierr);
ierr = VecNorm(work2,NORM_INFINITY,&test_err);CHKERRQ(ierr);
ierr = KSPComputeExtremeSingularValues(check_ksp,&lambda_max,&lambda_min);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(check_ksp,&k);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(pcbddc->dbg_viewer,"Subdomain %04d error for adapted KSP %1.14e (it %d, eigs %1.6e %1.6e)\n",PetscGlobalRank,test_err,k,lambda_min,lambda_max);CHKERRQ(ierr);
ierr = KSPDestroy(&check_ksp);CHKERRQ(ierr);
ierr = VecDestroy(&work1);CHKERRQ(ierr);
ierr = VecDestroy(&work2);CHKERRQ(ierr);
ierr = VecDestroy(&work3);CHKERRQ(ierr);
}
/* all processes shoud call this, even the void ones */
if (pcbddc->dbg_flag) {
ierr = PetscViewerFlush(pcbddc->dbg_viewer);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*! \brief solve complex use Krylov solver in combination with Block-Jacobi + Nested
*
*
* Solve the system using block-jacobi preconditioner + nested solver for each block
*
*/
void solve_complex(Mat & A_, const Vec & b_, Vec & x_)
{
// We get the size of the Matrix A
PetscInt row;
PetscInt col;
PetscInt row_loc;
PetscInt col_loc;
PetscInt * blks;
PetscInt nlocal;
PetscInt first;
KSP *subksp;
PC subpc;
PETSC_SAFE_CALL(MatGetSize(A_,&row,&col));
PETSC_SAFE_CALL(MatGetLocalSize(A_,&row_loc,&col_loc));
// Set the preconditioner to Block-jacobi
PC pc;
KSPGetPC(ksp,&pc);
PCSetType(pc,PCBJACOBI);
PETSC_SAFE_CALL(KSPSetType(ksp,KSPGMRES));
PetscInt m = 1;
PetscMalloc1(m,&blks);
for (size_t i = 0 ; i < m ; i++) blks[i] = row_loc;
PCBJacobiSetLocalBlocks(pc,m,blks);
PetscFree(blks);
KSPSetUp(ksp);
PCSetUp(pc);
PCBJacobiGetSubKSP(pc,&nlocal,&first,&subksp);
for (size_t i = 0; i < nlocal; i++)
{
KSPGetPC(subksp[i],&subpc);
PCSetType(subpc,PCLU);
// PCSetType(subpc,PCJACOBI);
KSPSetType(subksp[i],KSPPREONLY);
// PETSC_SAFE_CALL(KSPSetConvergenceTest(ksp,KSPConvergedDefault,NULL,NULL));
// KSPSetTolerances(subksp[i],1.e-6,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
/* if (!rank)
{
if (i%2)
{
PCSetType(subpc,PCILU);
}
else
{
PCSetType(subpc,PCNONE);
KSPSetType(subksp[i],KSPBCGS);
KSPSetTolerances(subksp[i],1.e-6,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
}
}
else
{
PCSetType(subpc,PCJACOBI);
KSPSetType(subksp[i],KSPGMRES);
KSPSetTolerances(subksp[i],1.e-6,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);
}*/
}
KSPSolve(ksp,b_,x_);
auto & v_cl = create_vcluster();
if (try_solve == true)
{
// calculate error statistic about the solution
solError err = statSolutionError(A_,b_,x_);
if (v_cl.getProcessUnitID() == 0)
{
std::cout << "Method: " << s_type << " " << " pre-conditoner: " << PCJACOBI << " iterations: " << err.its << std::endl;
std::cout << "Norm of error: " << err.err_norm << " Norm infinity: " << err.err_inf << std::endl;
}
}
}
PetscInt main(PetscInt argc,char **args)
{
Mat A,As;
PetscBool flg,disp_mat=PETSC_FALSE;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt i,j;
PetscScalar v,sigma2;
PetscRandom rctx;
PetscReal h2,sigma1=100.0;
PetscInt dim,Ii,J,n = 3,use_random,rstart,rend;
KSP ksp;
PC pc;
Mat F;
PetscInt nneg, nzero, npos;
PetscInitialize(&argc,&args,(char *)0,help);
#if !defined(PETSC_USE_COMPLEX)
SETERRQ(PETSC_COMM_WORLD,1,"This example requires complex numbers");
#endif
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL, "-display_mat", &disp_mat);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(PETSC_NULL,"-sigma1",&sigma1,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
dim = n*n;
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,dim,dim);CHKERRQ(ierr);
ierr = MatSetType(A,MATAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = PetscOptionsHasName(PETSC_NULL,"-norandom",&flg);CHKERRQ(ierr);
if (flg) use_random = 0;
else use_random = 1;
if (use_random) {
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(rctx,0.0,PETSC_i);CHKERRQ(ierr);
ierr = PetscRandomGetValue(rctx,&sigma2);CHKERRQ(ierr); /* RealPart(sigma2) == 0.0 */
} else {
sigma2 = 10.0*PETSC_i;
}
h2 = 1.0/((n+1)*(n+1));
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
for (Ii=rstart; Ii<rend; Ii++) {
v = -1.0; i = Ii/n; j = Ii - i*n;
if (i>0) {
J = Ii-n; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (i<n-1) {
J = Ii+n; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j>0) {
J = Ii-1; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j<n-1) {
J = Ii+1; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
v = 4.0 - sigma1*h2;
ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Check whether A is symmetric */
ierr = PetscOptionsHasName(PETSC_NULL, "-check_symmetric", &flg);CHKERRQ(ierr);
if (flg) {
Mat Trans;
ierr = MatTranspose(A,MAT_INITIAL_MATRIX, &Trans);
ierr = MatEqual(A, Trans, &flg);
if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"A is not symmetric");
ierr = MatDestroy(&Trans);CHKERRQ(ierr);
}
ierr = MatSetOption(A,MAT_SYMMETRIC,PETSC_TRUE);CHKERRQ(ierr);
/* make A complex Hermitian */
Ii = 0; J = dim-1;
if (Ii >= rstart && Ii < rend){
v = sigma2*h2; /* RealPart(v) = 0.0 */
ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);
v = -sigma2*h2;
ierr = MatSetValues(A,1,&J,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr);
}
Ii = dim-2; J = dim-1;
if (Ii >= rstart && Ii < rend){
v = sigma2*h2; /* RealPart(v) = 0.0 */
ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);
v = -sigma2*h2;
ierr = MatSetValues(A,1,&J,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Check whether A is Hermitian */
ierr = PetscOptionsHasName(PETSC_NULL, "-check_Hermitian", &flg);CHKERRQ(ierr);
if (flg) {
Mat Hermit;
if (disp_mat){
if (!rank) printf(" A:\n");
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = MatHermitianTranspose(A,MAT_INITIAL_MATRIX, &Hermit);
if (disp_mat){
if (!rank) printf(" A_Hermitian:\n");
ierr = MatView(Hermit,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = MatEqual(A, Hermit, &flg);
if (!flg) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"A is not Hermitian");
ierr = MatDestroy(&Hermit);CHKERRQ(ierr);
}
ierr = MatSetOption(A,MAT_HERMITIAN,PETSC_TRUE);CHKERRQ(ierr);
/* Create a Hermitian matrix As in sbaij format */
ierr = MatConvert(A,MATSBAIJ,MAT_INITIAL_MATRIX,&As);CHKERRQ(ierr);
if (disp_mat){
if (!rank) {ierr = PetscPrintf(PETSC_COMM_SELF," As:\n");CHKERRQ(ierr);}
ierr = MatView(As,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Test MatGetInertia() */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,As,As,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCCHOLESKY);CHKERRQ(ierr);
ierr = PCSetFromOptions(pc);CHKERRQ(ierr);
ierr = PCSetUp(pc);CHKERRQ(ierr);
ierr = PCFactorGetMatrix(pc,&F);CHKERRQ(ierr);
ierr = MatGetInertia(F,&nneg,&nzero,&npos);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (!rank){
ierr = PetscPrintf(PETSC_COMM_SELF," MatInertia: nneg: %D, nzero: %D, npos: %D\n",nneg,nzero,npos);CHKERRQ(ierr);
}
/* Free spaces */
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
if (use_random) {ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);}
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&As);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat A,B,F;
PetscErrorCode ierr;
KSP ksp;
PC pc;
PetscInt N, n=10, m, Istart, Iend, II, J, i,j;
PetscInt nneg, nzero, npos;
PetscScalar v,sigma;
PetscBool flag,loadA=PETSC_FALSE,loadB=PETSC_FALSE;
char file[2][PETSC_MAX_PATH_LEN];
PetscViewer viewer;
PetscMPIInt rank;
PetscInitialize(&argc,&args,(char *)0,help);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compute the matrices that define the eigensystem, Ax=kBx
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscOptionsGetString(PETSC_NULL,"-fA",file[0],PETSC_MAX_PATH_LEN,&loadA);CHKERRQ(ierr);
if (loadA) {
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file[0],FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetType(A,MATSBAIJ);CHKERRQ(ierr);
ierr = MatLoad(A,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-fB",file[1],PETSC_MAX_PATH_LEN,&loadB);CHKERRQ(ierr);
if (loadB){
/* load B to get A = A + sigma*B */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file[1],FILE_MODE_READ,&viewer);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr);
ierr = MatSetType(B,MATSBAIJ);CHKERRQ(ierr);
ierr = MatLoad(B,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
}
if (!loadA) { /* Matrix A is copied from slepc-3.0.0-p6/src/examples/ex13.c. */
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,&flag);CHKERRQ(ierr);
if( flag==PETSC_FALSE ) m=n;
N = n*m;
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,N,N);CHKERRQ(ierr);
ierr = MatSetType(A,MATSBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr);
for( II=Istart; II<Iend; II++ ) {
v = -1.0; i = II/n; j = II-i*n;
if(i>0) { J=II-n; MatSetValues(A,1,&II,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); }
if(i<m-1) { J=II+n; MatSetValues(A,1,&II,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); }
if(j>0) { J=II-1; MatSetValues(A,1,&II,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); }
if(j<n-1) { J=II+1; MatSetValues(A,1,&II,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); }
v=4.0; MatSetValues(A,1,&II,1,&II,&v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
if (!loadB) {
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(B,MATSBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatSetUp(B);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr);
for( II=Istart; II<Iend; II++ ) {
/* v=4.0; MatSetValues(B,1,&II,1,&II,&v,INSERT_VALUES);CHKERRQ(ierr); */
v=1.0; MatSetValues(B,1,&II,1,&II,&v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* ierr = MatView(B,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Set a shift: A = A - sigma*B */
ierr = PetscOptionsGetScalar(PETSC_NULL,"-sigma",&sigma,&flag);CHKERRQ(ierr);
if (flag){
sigma = -1.0 * sigma;
ierr = MatAXPY(A,sigma,B,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr); /* A <- A - sigma*B */
/* ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
}
/* Test MatGetInertia() */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCCHOLESKY);CHKERRQ(ierr);
ierr = PCSetFromOptions(pc);CHKERRQ(ierr);
ierr = PCSetUp(pc);CHKERRQ(ierr);
ierr = PCFactorGetMatrix(pc,&F);CHKERRQ(ierr);
ierr = MatGetInertia(F,&nneg,&nzero,&npos);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (!rank){
ierr = PetscPrintf(PETSC_COMM_SELF," MatInertia: nneg: %D, nzero: %D, npos: %D\n",nneg,nzero,npos);CHKERRQ(ierr);
}
/* Destroy */
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PETSC_EXTERN void PETSC_STDCALL pcsetup_(PC pc, int *__ierr ){
*__ierr = PCSetUp(
(PC)PetscToPointer((pc) ));
}
// =====================================================
void PetscPreconditioner::set_petsc_preconditioner_type
(const PreconditionerType & preconditioner_type, PC & pc, const int ¶llelOverlapping) {
int ierr = 0;
switch(preconditioner_type) {
case IDENTITY_PRECOND:
ierr = PCSetType(pc, (char*) PCNONE);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case CHOLESKY_PRECOND:
ierr = PCSetType(pc, (char*) PCCHOLESKY);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case ICC_PRECOND:
ierr = PCSetType(pc, (char*) PCICC);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case ILU_PRECOND:
{
int nprocs;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs); //TODO
// In serial, just set the ILU preconditioner type
if(nprocs == 1)
{
ierr = PCSetType(pc, (char*) PCILU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
}
else
{
// But PETSc has no truly parallel ILU, instead you have to set
// an actual parallel preconditioner (e.g. block Jacobi (parlleloverlapping 0) or ASM (parlleloverlapping >0))
// and then assign ILU sub-preconditioners.
set_petsc_preconditioner_type(ASM_PRECOND, pc);
PCASMSetOverlap(pc, parallelOverlapping);
PCSetUp(pc);
// Set ILU as the sub preconditioner type
set_petsc_subpreconditioner_type(PCILU, pc);
}
break;
}
case LU_PRECOND: {
int nprocs;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
if(nprocs == 1) {
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
}
else {
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERMUMPS);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetUpMatSolverPackage(pc);
CHKERRABORT(MPI_COMM_WORLD, ierr);
Mat F;
ierr = PCFactorGetMatrix(pc, &F);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = MatMumpsSetIcntl(F, 14, 30);
CHKERRABORT(MPI_COMM_WORLD, ierr);
}
break;
}
case SLU_PRECOND:
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERSUPERLU_DIST);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break; //here we set the SuperLU_dist solver package
case MLU_PRECOND: //here we set the MUMPS parallel direct solver package
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERMUMPS);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetUpMatSolverPackage(pc);
CHKERRABORT(MPI_COMM_WORLD, ierr);
Mat F;
ierr = PCFactorGetMatrix(pc, &F);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = MatMumpsSetIcntl(F, 14, 30);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case ULU_PRECOND: //here we set the Umfpack serial direct solver package
ierr = PCSetType(pc, (char*) PCLU);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERUMFPACK);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetUpMatSolverPackage(pc);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case MCC_PRECOND:
ierr = PCSetType(pc, (char*) PCCHOLESKY);
CHKERRABORT(MPI_COMM_WORLD, ierr);
ierr = PCFactorSetMatSolverPackage(pc, MATSOLVERMUMPS);
CHKERRABORT(MPI_COMM_WORLD, ierr); //here we set the MUMPS parallel direct solver package
break;
case ASM_PRECOND:
ierr = PCSetType(pc, (char*) PCASM);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case FIELDSPLIT_PRECOND:
ierr = PCSetType(pc, (char*) PCFIELDSPLIT);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case JACOBI_PRECOND:
ierr = PCSetType(pc, (char*) PCJACOBI);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case BLOCK_JACOBI_PRECOND:
ierr = PCSetType(pc, (char*) PCBJACOBI);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case SOR_PRECOND:
ierr = PCSetType(pc, (char*) PCSOR);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case EISENSTAT_PRECOND:
ierr = PCSetType(pc, (char*) PCEISENSTAT);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case AMG_PRECOND:
ierr = PCSetType(pc, (char*) PCHYPRE);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case MG_PRECOND:
ierr = PCSetType(pc, (char*) PCMG);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
case LSC_PRECOND:
ierr = PCSetType(pc, (char*) PCLSC);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
#if !(PETSC_VERSION_LESS_THAN(2,1,2))
// Only available for PETSC >= 2.1.2
case USER_PRECOND:
ierr = PCSetType(pc, (char*) PCMAT);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
#endif
case SHELL_PRECOND:
ierr = PCSetType(pc, (char*) PCSHELL);
CHKERRABORT(MPI_COMM_WORLD, ierr);
break;
default:
std::cerr << "ERROR: Unsupported PETSC Preconditioner: "
<< preconditioner_type << std::endl
<< "Continuing with PETSC defaults" << std::endl;
}
//Let the commandline override stuff
if(preconditioner_type != AMG_PRECOND && preconditioner_type != MG_PRECOND) PCSetFromOptions(pc); //!!!!!!
}
PetscErrorCode FluidFieldSetup( FluidField f )
{
PetscLogDouble t1,t2;
PetscErrorCode ierr;
PetscFunctionBegin;
// Assemble viscous matricies
ierr = FluidFieldMatAssemble( f ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Lengths: %e %e %e\n", f->lens.x, f->lens.y, f->lens.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Size: %d %d %d\n", f->dims.x, f->dims.y, f->dims.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "dx: %e %e %e\n", f->dh.x, f->dh.y, f->dh.z ); CHKERRQ(ierr);
ierr = PetscTime(&t1); CHKERRQ(ierr);
// Create vectors
ierr = GACreate( f->daV, &f->ga); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daV,&f->rhs); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel0); CHKERRQ(ierr);
// ierr = DACreateGlobalVector(f->daE,&f->E); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daB,&f->buf); CHKERRQ(ierr);
// Set up the outer solver
ierr = KSPCreate(f->comm,&f->ksp); CHKERRQ(ierr);
ierr = KSPSetOperators(f->ksp,f->mat,f->mat, SAME_PRECONDITIONER); CHKERRQ(ierr);
ierr = KSPSetType(f->ksp,KSPFGMRES); CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(f->ksp,PETSC_TRUE); CHKERRQ(ierr);
ierr = KSPSetTolerances(f->ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr);
ierr = KSPSetFromOptions(f->ksp);CHKERRQ(ierr);
// Split pressure from velocity [ u v w | p ]
PC pc;
ierr = KSPGetPC(f->ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
if( f->is3D ) {
const PetscInt ufields[] = {U_FACE,V_FACE,W_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,4); CHKERRQ(ierr); // [p u v w]
ierr = PCFieldSplitSetFields(pc,"v",3,ufields,ufields); CHKERRQ(ierr); // [u v w]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
} else {
const PetscInt ufields[] = {U_FACE,V_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr); // [p u v]
ierr = PCFieldSplitSetFields(pc,"v",2,ufields,ufields); CHKERRQ(ierr); // [u v]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
}
ierr = PCSetUp(pc); CHKERRQ(ierr);
int nVelP;
KSP *kspVelP;
ierr = PCFieldSplitGetSubKSP(pc,&nVelP,&kspVelP); CHKERRQ(ierr);
ierr = KSPSetTolerances(kspVelP[1],PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,4); CHKERRQ(ierr);
ierr = KSPSetType(kspVelP[1],KSPGMRES); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[1],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspVelP[1]);CHKERRQ(ierr);
// Split velocity [u v w] into component matricies [u], [v], [w]
ierr = KSPSetType(kspVelP[0],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[0],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_ADDITIVE); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,f->is3D?3:2); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
/* Set solver for each velocity component
* Split component velocity as parallel blocks along processors
* Use direct solver for each block
* TODO: use MG, w/FFT on coarse grid
*/
ierr = PetscTime(&t2); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Finished Solver Setup: %f sec\n",t2-t1); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char** argv)
{
/* the total number of grid points in each spatial direction is (n+1) */
/* the total number of degrees-of-freedom in each spatial direction is (n-1) */
/* this version requires n to be a power of 2 */
if( argc < 2 ) {
printf("need a problem size\n");
return 1;
}
int n = atoi(argv[1]);
int m = n-1;
// Initialize Petsc
PetscInitialize(&argc,&argv,0,PETSC_NULL);
// Create our vector
Vec b;
VecCreate(PETSC_COMM_WORLD,&b);
VecSetSizes(b,m*m,PETSC_DECIDE);
VecSetFromOptions(b);
VecSetUp(b);
// Create our matrix
Mat A;
MatCreate(PETSC_COMM_WORLD,&A);
MatSetType(A,MATSEQAIJ);
MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,m*m,m*m);
MatSetUp(A);
// Create linear solver object
KSP ksp;
KSPCreate(PETSC_COMM_WORLD,&ksp);
// setup rhs
PetscInt low, high;
VecGetOwnershipRange(b,&low,&high);
PetscInt* inds = (PetscInt*)malloc((high-low)*sizeof(PetscInt));
double* vals = (double*)malloc((high-low)*sizeof(double));
double h = 1./(double)n;
for (int i=0;i<high-1;++i) {
inds[i] = low+i;
vals[i] = h*h;
}
VecSetValues(b,high-low,inds,vals,INSERT_VALUES);
free(inds);
free(vals);
// setup matrix
for (int i=0;i<m*m;++i) {
MatSetValue(A,i,i,4.f,INSERT_VALUES);
if (i%m != m-1)
MatSetValue(A,i,i+1,-1.f,INSERT_VALUES);
if (i%m)
MatSetValue(A,i,i-1,-1.f,INSERT_VALUES);
if (i > m)
MatSetValue(A,i,i-m,-1.f,INSERT_VALUES);
if (i < m*(m-1))
MatSetValue(A,i,i+m,-1.f,INSERT_VALUES);
}
// sync processes
VecAssemblyBegin(b);
VecAssemblyEnd(b);
MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
// solve
KSPSetType(ksp,"cg");
KSPSetTolerances(ksp,1.e-10,1.e-10,1.e6,10000);
KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);
PC pc;
KSPGetPC(ksp,&pc);
PCSetType(pc,"ilu");
PCSetFromOptions(pc);
PCSetUp(pc);
// setup solver
KSPSetFromOptions(ksp);
KSPSetUp(ksp);
Vec x;
VecDuplicate(b,&x);
KSPSolve(ksp,b,x);
double val;
VecNorm(x,NORM_INFINITY,&val);
printf (" umax = %e \n",val);
PetscFinalize();
return 0;
}
KSP ksp;
ierr = KSPCreate(comm,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY); CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,mat,mat,DIFFERENT_NONZERO_PATTERN); CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
//Split pressure from velocity
PC pc;
ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr);
ierr = PCFieldSplitSetFields(pc,2,(int[]){0,1}); CHKERRQ(ierr);
ierr = PCFieldSplitSetFields(pc,1,(int[]){2}); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
//Split velocity into u-v component matricies
int nVelP;
KSP *kspVelP;
ierr = PCFieldSplitGetSubKSP(pc,&nVelP,&kspVelP); CHKERRQ(ierr);
ierr = KSPSetTolerances(kspVelP[1],1e-50,1e-3,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr);
// ierr = KSPSetType(kspVelP[1],KSPCG); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[1],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
ierr = KSPSetType(kspVelP[0],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[0],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_ADDITIVE); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,2); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
PetscErrorCode KSPSolve_FBCGSR(KSP ksp)
{
PetscErrorCode ierr;
PetscInt i,j,N;
PetscScalar tau,sigma,alpha,omega,beta;
PetscReal rho;
PetscScalar xi1,xi2,xi3,xi4;
Vec X,B,P,P2,RP,R,V,S,T,S2;
PetscScalar *PETSC_RESTRICT rp, *PETSC_RESTRICT r, *PETSC_RESTRICT p;
PetscScalar *PETSC_RESTRICT v, *PETSC_RESTRICT s, *PETSC_RESTRICT t, *PETSC_RESTRICT s2;
PetscScalar insums[4],outsums[4];
KSP_BCGS *bcgs = (KSP_BCGS*)ksp->data;
PC pc;
PetscFunctionBegin;
if (!ksp->vec_rhs->petscnative) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"Only coded for PETSc vectors");
ierr = VecGetLocalSize(ksp->vec_sol,&N);
CHKERRQ(ierr);
X = ksp->vec_sol;
B = ksp->vec_rhs;
P2 = ksp->work[0];
/* The followings are involved in modified inner product calculations and vector updates */
RP = ksp->work[1];
ierr = VecGetArray(RP,(PetscScalar**)&rp);
CHKERRQ(ierr);
ierr = VecRestoreArray(RP,NULL);
CHKERRQ(ierr);
R = ksp->work[2];
ierr = VecGetArray(R,(PetscScalar**)&r);
CHKERRQ(ierr);
ierr = VecRestoreArray(R,NULL);
CHKERRQ(ierr);
P = ksp->work[3];
ierr = VecGetArray(P,(PetscScalar**)&p);
CHKERRQ(ierr);
ierr = VecRestoreArray(P,NULL);
CHKERRQ(ierr);
V = ksp->work[4];
ierr = VecGetArray(V,(PetscScalar**)&v);
CHKERRQ(ierr);
ierr = VecRestoreArray(V,NULL);
CHKERRQ(ierr);
S = ksp->work[5];
ierr = VecGetArray(S,(PetscScalar**)&s);
CHKERRQ(ierr);
ierr = VecRestoreArray(S,NULL);
CHKERRQ(ierr);
T = ksp->work[6];
ierr = VecGetArray(T,(PetscScalar**)&t);
CHKERRQ(ierr);
ierr = VecRestoreArray(T,NULL);
CHKERRQ(ierr);
S2 = ksp->work[7];
ierr = VecGetArray(S2,(PetscScalar**)&s2);
CHKERRQ(ierr);
ierr = VecRestoreArray(S2,NULL);
CHKERRQ(ierr);
/* Only supports right preconditioning */
if (ksp->pc_side != PC_RIGHT) SETERRQ1(PetscObjectComm((PetscObject)ksp),PETSC_ERR_SUP,"KSP fbcgsr does not support %s",PCSides[ksp->pc_side]);
if (!ksp->guess_zero) {
if (!bcgs->guess) {
ierr = VecDuplicate(X,&bcgs->guess);
CHKERRQ(ierr);
}
ierr = VecCopy(X,bcgs->guess);
CHKERRQ(ierr);
} else {
ierr = VecSet(X,0.0);
CHKERRQ(ierr);
}
/* Compute initial residual */
ierr = KSPGetPC(ksp,&pc);
CHKERRQ(ierr);
ierr = PCSetUp(pc);
CHKERRQ(ierr);
if (!ksp->guess_zero) {
ierr = MatMult(pc->mat,X,P2);
CHKERRQ(ierr); /* P2 is used as temporary storage */
ierr = VecCopy(B,R);
CHKERRQ(ierr);
ierr = VecAXPY(R,-1.0,P2);
CHKERRQ(ierr);
} else {
ierr = VecCopy(B,R);
CHKERRQ(ierr);
}
/* Test for nothing to do */
if (ksp->normtype != KSP_NORM_NONE) {
ierr = VecNorm(R,NORM_2,&rho);
CHKERRQ(ierr);
}
ierr = PetscObjectSAWsTakeAccess((PetscObject)ksp);
CHKERRQ(ierr);
ksp->its = 0;
ksp->rnorm = rho;
ierr = PetscObjectSAWsGrantAccess((PetscObject)ksp);
CHKERRQ(ierr);
ierr = KSPLogResidualHistory(ksp,rho);
CHKERRQ(ierr);
ierr = KSPMonitor(ksp,0,rho);
CHKERRQ(ierr);
ierr = (*ksp->converged)(ksp,0,rho,&ksp->reason,ksp->cnvP);
CHKERRQ(ierr);
if (ksp->reason) PetscFunctionReturn(0);
/* Initialize iterates */
ierr = VecCopy(R,RP);
CHKERRQ(ierr); /* rp <- r */
ierr = VecCopy(R,P);
CHKERRQ(ierr); /* p <- r */
/* Big loop */
for (i=0; i<ksp->max_it; i++) {
/* matmult and pc */
ierr = PCApply(pc,P,P2);
CHKERRQ(ierr); /* p2 <- K p */
ierr = MatMult(pc->mat,P2,V);
CHKERRQ(ierr); /* v <- A p2 */
/* inner prodcuts */
if (i==0) {
tau = rho*rho;
ierr = VecDot(V,RP,&sigma);
CHKERRQ(ierr); /* sigma <- (v,rp) */
} else {
ierr = PetscLogEventBegin(VEC_ReduceArithmetic,0,0,0,0);
CHKERRQ(ierr);
tau = sigma = 0.0;
for (j=0; j<N; j++) {
tau += r[j]*rp[j]; /* tau <- (r,rp) */
sigma += v[j]*rp[j]; /* sigma <- (v,rp) */
}
PetscLogFlops(4.0*N);
ierr = PetscLogEventEnd(VEC_ReduceArithmetic,0,0,0,0);
CHKERRQ(ierr);
insums[0] = tau;
insums[1] = sigma;
ierr = PetscLogEventBarrierBegin(VEC_ReduceBarrier,0,0,0,0,PetscObjectComm((PetscObject)ksp));
CHKERRQ(ierr);
ierr = MPI_Allreduce(insums,outsums,2,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)ksp));
CHKERRQ(ierr);
ierr = PetscLogEventBarrierEnd(VEC_ReduceBarrier,0,0,0,0,PetscObjectComm((PetscObject)ksp));
CHKERRQ(ierr);
tau = outsums[0];
sigma = outsums[1];
}
/* scalar update */
alpha = tau / sigma;
/* vector update */
ierr = VecWAXPY(S,-alpha,V,R);
CHKERRQ(ierr); /* s <- r - alpha v */
/* matmult and pc */
ierr = PCApply(pc,S,S2);
CHKERRQ(ierr); /* s2 <- K s */
ierr = MatMult(pc->mat,S2,T);
CHKERRQ(ierr); /* t <- A s2 */
/* inner prodcuts */
ierr = PetscLogEventBegin(VEC_ReduceArithmetic,0,0,0,0);
CHKERRQ(ierr);
xi1 = xi2 = xi3 = xi4 = 0.0;
for (j=0; j<N; j++) {
xi1 += s[j]*s[j]; /* xi1 <- (s,s) */
xi2 += t[j]*s[j]; /* xi2 <- (t,s) */
xi3 += t[j]*t[j]; /* xi3 <- (t,t) */
xi4 += t[j]*rp[j]; /* xi4 <- (t,rp) */
}
PetscLogFlops(8.0*N);
ierr = PetscLogEventEnd(VEC_ReduceArithmetic,0,0,0,0);
CHKERRQ(ierr);
insums[0] = xi1;
insums[1] = xi2;
insums[2] = xi3;
insums[3] = xi4;
ierr = PetscLogEventBarrierBegin(VEC_ReduceBarrier,0,0,0,0,PetscObjectComm((PetscObject)ksp));
CHKERRQ(ierr);
ierr = MPI_Allreduce(insums,outsums,4,MPIU_SCALAR,MPIU_SUM,PetscObjectComm((PetscObject)ksp));
CHKERRQ(ierr);
ierr = PetscLogEventBarrierEnd(VEC_ReduceBarrier,0,0,0,0,PetscObjectComm((PetscObject)ksp));
CHKERRQ(ierr);
xi1 = outsums[0];
xi2 = outsums[1];
xi3 = outsums[2];
xi4 = outsums[3];
/* test denominator */
if (xi3 == 0.0) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_PLIB,"Divide by zero");
if (sigma == 0.0) SETERRQ(PetscObjectComm((PetscObject)ksp),PETSC_ERR_PLIB,"Divide by zero");
/* scalar updates */
omega = xi2 / xi3;
beta = -xi4 / sigma;
rho = PetscSqrtReal(PetscAbsScalar(xi1 - omega * xi2)); /* residual norm */
/* vector updates */
ierr = VecAXPBYPCZ(X,alpha,omega,1.0,P2,S2);
CHKERRQ(ierr); /* x <- alpha * p2 + omega * s2 + x */
/* convergence test */
ierr = PetscObjectSAWsTakeAccess((PetscObject)ksp);
CHKERRQ(ierr);
ksp->its++;
ksp->rnorm = rho;
ierr = PetscObjectSAWsGrantAccess((PetscObject)ksp);
CHKERRQ(ierr);
ierr = KSPLogResidualHistory(ksp,rho);
CHKERRQ(ierr);
ierr = KSPMonitor(ksp,i+1,rho);
CHKERRQ(ierr);
ierr = (*ksp->converged)(ksp,i+1,rho,&ksp->reason,ksp->cnvP);
CHKERRQ(ierr);
if (ksp->reason) break;
/* vector updates */
ierr = PetscLogEventBegin(VEC_Ops,0,0,0,0);
CHKERRQ(ierr);
for (j=0; j<N; j++) {
r[j] = s[j] - omega * t[j]; /* r <- s - omega t */
p[j] = r[j] + beta * (p[j] - omega * v[j]); /* p <- r + beta * (p - omega v) */
}
PetscLogFlops(6.0*N);
ierr = PetscLogEventEnd(VEC_Ops,0,0,0,0);
CHKERRQ(ierr);
}
if (i >= ksp->max_it) ksp->reason = KSP_DIVERGED_ITS;
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat mat; /* matrix */
Vec b,ustar,u; /* vectors (RHS, exact solution, approx solution) */
PC pc; /* PC context */
KSP ksp; /* KSP context */
PetscErrorCode ierr;
PetscInt n = 10,i,its,col[3];
PetscScalar value[3];
PCType pcname;
KSPType kspname;
PetscReal norm,tol=1000.*PETSC_MACHINE_EPSILON;
PetscInitialize(&argc,&args,(char*)0,help);
/* Create and initialize vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&b);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&ustar);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&u);CHKERRQ(ierr);
ierr = VecSet(ustar,1.0);CHKERRQ(ierr);
ierr = VecSet(u,0.0);CHKERRQ(ierr);
/* Create and assemble matrix */
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,n,n,3,NULL,&mat);CHKERRQ(ierr);
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(mat,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0] = n - 2; col[1] = n - 1;
ierr = MatSetValues(mat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0; col[0] = 0; col[1] = 1; value[0] = 2.0; value[1] = -1.0;
ierr = MatSetValues(mat,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Compute right-hand-side vector */
ierr = MatMult(mat,ustar,b);CHKERRQ(ierr);
/* Create PC context and set up data structures */
ierr = PCCreate(PETSC_COMM_WORLD,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE);CHKERRQ(ierr);
ierr = PCSetFromOptions(pc);CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat);CHKERRQ(ierr);
ierr = PCSetUp(pc);CHKERRQ(ierr);
/* Create KSP context and set up data structures */
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPRICHARDSON);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = PCSetOperators(pc,mat,mat);CHKERRQ(ierr);
ierr = KSPSetPC(ksp,pc);CHKERRQ(ierr);
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
/* Solve the problem */
ierr = KSPGetType(ksp,&kspname);CHKERRQ(ierr);
ierr = PCGetType(pc,&pcname);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"Running %s with %s preconditioning\n",kspname,pcname);CHKERRQ(ierr);
ierr = KSPSolve(ksp,b,u);CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,ustar);CHKERRQ(ierr);
ierr = VecNorm(u,NORM_2,&norm);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"2 norm of error %g Number of iterations %D\n",(double)norm,its);CHKERRQ(ierr);
}
/* Free data structures */
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&ustar);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = PCDestroy(&pc);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
END_TEST
START_TEST( test_IIMContext )
{
IIM iim;
int d1=63, d2=d1;
Grid2D rhsC, p, px, py, u, v;
LevelSet2D ls, lstemp;
PetscErrorCode ierr;
Mat m;
KSP ksp;
PC pc;
Generate2DLaplacianPeriodicBC( d1, d2, &m);
ierr = KSPCreate(PETSC_COMM_SELF, &ksp); CHKERRQ(ierr);
KSPGetPC(ksp, &pc);
// KSPSetType(ksp, KSPCG);
// PCSetType(pc, PCICC);
// PCFactorSetLevels(pc, 0);
// KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
KSPSetType(ksp, KSPPREONLY);
PCSetType(pc, PCCHOLESKY);
PCFactorSetMatOrderingType(pc, MATORDERING_ND);
KSPSetFromOptions(ksp);
KSPSetOperators(ksp, m, m, SAME_PRECONDITIONER);
PCSetUp(pc);
ierr = IIMCreate(&iim, 12); CHKERRQ(ierr);
// IIMSetForceComponents(iim, ForceComponentNormalSimple, ForceComponentTangentialSimple);
CreateGrid2D(d1, d2, &rhsC);
CreateGrid2D(d1, d2, &p);
CreateGrid2D(d1, d2, &px);
CreateGrid2D(d1, d2, &py);
CreateGrid2D(d1, d2, &u);
CreateGrid2D(d1, d2, &v);
ierr = CreateLevelSet2D(d1,d2,&ls); CHKERRQ(ierr);
CreateLevelSet2D(d1, d2, &lstemp);
LevelSetInitializeToStar(ls,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE);
// LevelSetInitializeToCircle(ls, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE);
Vec u2,v2,magVel;
VecCreateSeq(PETSC_COMM_SELF,d1*d2,&magVel);
VecDuplicate(magVel, &u2);
VecDuplicate(magVel, &v2);
PetscReal maxVel, dt;
WriteVec wv, wvC, wvP, wvPX, wvPY, wvU, wvV;
WriteVecCreate(lstemp->g2d->v, "ls",&wv);
WriteVecCreate(rhsC->v, "c", &wvC);
WriteVecCreate(p->v, "p", &wvP);
WriteVecCreate(px->v, "px", &wvPY);
WriteVecCreate(py->v, "py", &wvPX);
WriteVecCreate(u->v, "u", &wvU);
WriteVecCreate(v->v, "v", &wvV);
for( int t = 0; t < 100; t++ )
{
printf("TIME: %d ",t);
VecSet(rhsC->v, 0.); // TODO: this is important when reusing vecs, need to reset back to zero after time step
ierr = IIMComputeCorrection2D(iim,JumpConditionPressure,ls,rhsC); CHKERRQ(ierr);
IIMDiscreteCompatabilityCondition(ls, rhsC);
ierr = KSPSolve(ksp, rhsC->v, p->v); CHKERRQ(ierr);
IrregularNodeListWrite(ls,t);
ierr = IIMPressureGradient(ls,p,px,py); CHKERRQ(ierr);
ierr = IIMComputeCorrection2D(iim,JumpConditionXVelocity,ls,px); CHKERRQ(ierr);
px->v2[d1/2][d2/2] = 0.;
ierr = KSPSolve(ksp, px->v, u->v); CHKERRQ(ierr);
ierr = IIMComputeCorrection2D(iim,JumpConditionYVelocity,ls,py); CHKERRQ(ierr);
py->v2[d1/2][d2/2] = 0.;
ierr = KSPSolve(ksp, py->v, v->v); CHKERRQ(ierr);
VecPointwiseMult(u2, u->v, u->v);
VecPointwiseMult(v2, u->v, u->v);
VecWAXPY(magVel, 1, u2, v2);
VecSqrt( magVel );
VecNorm(magVel, NORM_INFINITY, &maxVel);
dt = PetscMin(1/(2*maxVel),.1);
LevelSetAdvect2D(dt, u, v, ls, lstemp);
printf("\tmaxVel: %f",maxVel);
printf("\tdt: %f\n",dt);
WriteVecToDisk(wv);
WriteVecToDisk(wvC);
WriteVecToDisk(wvP);
WriteVecToDisk(wvPX);
WriteVecToDisk(wvPY);
WriteVecToDisk(wvU);
WriteVecToDisk(wvV);
VecCopy( lstemp->g2d->v, ls->g2d->v);
UpdateIrregularNodeList(ls);
// ReinitializeLevelSet(ls);
}
ierr = VecDestroy(magVel); CHKERRQ(ierr);
ierr = VecDestroy(u2); CHKERRQ(ierr);
ierr = VecDestroy(v2); CHKERRQ(ierr);
ierr = DestroyLevelSet2D(ls); CHKERRQ(ierr);
ierr = DestroyGrid2D(rhsC); CHKERRQ(ierr);
ierr = DestroyGrid2D(p); CHKERRQ(ierr);
ierr = DestroyGrid2D(px); CHKERRQ(ierr);
ierr = DestroyGrid2D(py); CHKERRQ(ierr);
ierr = DestroyGrid2D(u); CHKERRQ(ierr);
ierr = DestroyGrid2D(v); CHKERRQ(ierr);
ierr = IIMDestroy(iim); CHKERRQ(ierr);
}
