int main(int argc, char **args)
{
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &args, (char *) 0, ""); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Start %s\n", __FILE__); CHKERRQ(ierr);
MPI_Comm comm = PETSC_COMM_WORLD;
DA da;
Mat mat;
int MX = 100;
int dof = 3;
ierr = PetscPrintf(comm,"Started Assembly\n"); CHKERRQ(ierr);
ierr = MatMake(MX,dof,&da, &mat); CHKERRQ(ierr);
ierr = MatWrite("J",mat); CHKERRQ(ierr);
ierr = PetscPrintf(comm,"Finished Assembly\n"); CHKERRQ(ierr);
Vec rhs, sol;
ierr = DACreateGlobalVector(da,&rhs); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da,&sol); CHKERRQ(ierr);
int i,j;
int xs,ys,xm,ym;
Field **array;
ierr = DAVecGetArray(da,rhs,&array); CHKERRQ(ierr);
ierr = DAGetCorners(da,&xs,&ys,PETSC_NULL,&xm,&ym,PETSC_NULL); CHKERRQ(ierr);
for (j = xs; j < xm; ++j) {
for (i = ys; i < ym; ++i) {
if( 3*MX/4 > i && i > MX/4 &&
3*MX/4 > j && j > MX/4 )
{
array[j][i].u = 100.;
array[j][i].v = 100.;
}
}
}
ierr = DAVecRestoreArray(da,rhs,&array); CHKERRQ(ierr);
KSP ksp;
ierr = KSPCreate(comm,&ksp); CHKERRQ(ierr);
ierr = KSPSetType(ksp,KSPPREONLY); CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,mat,mat,DIFFERENT_NONZERO_PATTERN); CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
//Split pressure from velocity
PC pc;
ierr = KSPGetPC(ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr);
ierr = PCFieldSplitSetFields(pc,2,(int[]){0,1}); CHKERRQ(ierr);
PETSC_EXTERN void PETSC_STDCALL pcfieldsplitsetblocksize_(PC pc,PetscInt *bs, int *__ierr ){
*__ierr = PCFieldSplitSetBlockSize(
(PC)PetscToPointer((pc) ),*bs);
}
PetscErrorCode port_lsd_bfbt(void)
{
Mat A;
Vec x,b;
KSP ksp_A;
PC pc_A;
IS isu,isp;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = LoadTestMatrices(&A,&x,&b,&isu,&isp);CHKERRQ(ierr);
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp_A);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(ksp_A,"fc_");CHKERRQ(ierr);
ierr = KSPSetOperators(ksp_A,A,A);CHKERRQ(ierr);
ierr = KSPGetPC(ksp_A,&pc_A);CHKERRQ(ierr);
ierr = PCSetType(pc_A,PCFIELDSPLIT);CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc_A,2);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc_A,"velocity",isu);CHKERRQ(ierr);
ierr = PCFieldSplitSetIS(pc_A,"pressure",isp);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp_A);CHKERRQ(ierr);
ierr = KSPSolve(ksp_A,b,x);CHKERRQ(ierr);
/* Pull u,p out of x */
{
PetscInt loc;
PetscReal max,norm;
PetscScalar sum;
Vec uvec,pvec;
VecScatter uscat,pscat;
Mat A11,A22;
/* grab matrices and create the compatable u,p vectors */
ierr = MatGetSubMatrix(A,isu,isu,MAT_INITIAL_MATRIX,&A11);CHKERRQ(ierr);
ierr = MatGetSubMatrix(A,isp,isp,MAT_INITIAL_MATRIX,&A22);CHKERRQ(ierr);
ierr = MatCreateVecs(A11,&uvec,NULL);CHKERRQ(ierr);
ierr = MatCreateVecs(A22,&pvec,NULL);CHKERRQ(ierr);
/* perform the scatter from x -> (u,p) */
ierr = VecScatterCreate(x,isu,uvec,NULL,&uscat);CHKERRQ(ierr);
ierr = VecScatterBegin(uscat,x,uvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(uscat,x,uvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterCreate(x,isp,pvec,NULL,&pscat);CHKERRQ(ierr);
ierr = VecScatterBegin(pscat,x,pvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(pscat,x,pvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- vector vector values --\n");
ierr = VecMin(uvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(u) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(uvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(u) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(uvec,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(u) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(uvec,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(u) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- pressure vector values --\n");
ierr = VecMin(pvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(pvec,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(pvec,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(p) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(pvec,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(p) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"-- Full vector values --\n");
ierr = VecMin(x,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Min(u,p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecMax(x,&loc,&max);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Max(u,p) = %1.6f [loc=%D]\n",(double)max,loc);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Norm(u,p) = %1.6f \n",(double)norm);CHKERRQ(ierr);
ierr = VecSum(x,&sum);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Sum(u,p) = %1.6f \n",(double)PetscRealPart(sum));CHKERRQ(ierr);
ierr = VecScatterDestroy(&uscat);CHKERRQ(ierr);
ierr = VecScatterDestroy(&pscat);CHKERRQ(ierr);
ierr = VecDestroy(&uvec);CHKERRQ(ierr);
ierr = VecDestroy(&pvec);CHKERRQ(ierr);
ierr = MatDestroy(&A11);CHKERRQ(ierr);
ierr = MatDestroy(&A22);CHKERRQ(ierr);
}
ierr = KSPDestroy(&ksp_A);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = ISDestroy(&isu);CHKERRQ(ierr);
ierr = ISDestroy(&isp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode FluidFieldSetup( FluidField f )
{
PetscLogDouble t1,t2;
PetscErrorCode ierr;
PetscFunctionBegin;
// Assemble viscous matricies
ierr = FluidFieldMatAssemble( f ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Lengths: %e %e %e\n", f->lens.x, f->lens.y, f->lens.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Size: %d %d %d\n", f->dims.x, f->dims.y, f->dims.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "dx: %e %e %e\n", f->dh.x, f->dh.y, f->dh.z ); CHKERRQ(ierr);
ierr = PetscTime(&t1); CHKERRQ(ierr);
// Create vectors
ierr = GACreate( f->daV, &f->ga); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daV,&f->rhs); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel0); CHKERRQ(ierr);
// ierr = DACreateGlobalVector(f->daE,&f->E); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daB,&f->buf); CHKERRQ(ierr);
// Set up the outer solver
ierr = KSPCreate(f->comm,&f->ksp); CHKERRQ(ierr);
ierr = KSPSetOperators(f->ksp,f->mat,f->mat, SAME_PRECONDITIONER); CHKERRQ(ierr);
ierr = KSPSetType(f->ksp,KSPFGMRES); CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(f->ksp,PETSC_TRUE); CHKERRQ(ierr);
ierr = KSPSetTolerances(f->ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr);
ierr = KSPSetFromOptions(f->ksp);CHKERRQ(ierr);
// Split pressure from velocity [ u v w | p ]
PC pc;
ierr = KSPGetPC(f->ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
if( f->is3D ) {
const PetscInt ufields[] = {U_FACE,V_FACE,W_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,4); CHKERRQ(ierr); // [p u v w]
ierr = PCFieldSplitSetFields(pc,"v",3,ufields,ufields); CHKERRQ(ierr); // [u v w]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
} else {
const PetscInt ufields[] = {U_FACE,V_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr); // [p u v]
ierr = PCFieldSplitSetFields(pc,"v",2,ufields,ufields); CHKERRQ(ierr); // [u v]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
}
ierr = PCSetUp(pc); CHKERRQ(ierr);
int nVelP;
KSP *kspVelP;
ierr = PCFieldSplitGetSubKSP(pc,&nVelP,&kspVelP); CHKERRQ(ierr);
ierr = KSPSetTolerances(kspVelP[1],PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,4); CHKERRQ(ierr);
ierr = KSPSetType(kspVelP[1],KSPGMRES); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[1],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspVelP[1]);CHKERRQ(ierr);
// Split velocity [u v w] into component matricies [u], [v], [w]
ierr = KSPSetType(kspVelP[0],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[0],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_ADDITIVE); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,f->is3D?3:2); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
/* Set solver for each velocity component
* Split component velocity as parallel blocks along processors
* Use direct solver for each block
* TODO: use MG, w/FFT on coarse grid
*/
ierr = PetscTime(&t2); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Finished Solver Setup: %f sec\n",t2-t1); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
ierr = PCFieldSplitSetFields(pc,1,(int[]){2}); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
//Split velocity into u-v component matricies
int nVelP;
KSP *kspVelP;
ierr = PCFieldSplitGetSubKSP(pc,&nVelP,&kspVelP); CHKERRQ(ierr);
ierr = KSPSetTolerances(kspVelP[1],1e-50,1e-3,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr);
// ierr = KSPSetType(kspVelP[1],KSPCG); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[1],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
ierr = KSPSetType(kspVelP[0],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[0],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_ADDITIVE); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,2); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
//Set solver for each velocity component
int nVel;
KSP *kspVel;
ierr = PCFieldSplitGetSubKSP(pc,&nVel,&kspVel); CHKERRQ(ierr);
for( i = 0; i < nVel; i++ ) {
// Direct Method
ierr = KSPSetType(kspVel[i],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVel[i],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCCHOLESKY); CHKERRQ(ierr);
ierr = PCFactorSetMatOrderingType(pc,MATORDERING_ND); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
//
/* Iterative Method
