PetscErrorCode TestMatZeroRows(Mat A, Mat Afull, PetscBool squaretest, IS is, PetscScalar diag)
{
Mat B,Bcheck,B2 = NULL,lB;
Vec x = NULL, b = NULL, b2 = NULL;
ISLocalToGlobalMapping l2gr,l2gc;
PetscReal error;
char diagstr[16];
const PetscInt *idxs;
PetscInt rst,ren,i,n,N,d;
PetscMPIInt rank;
PetscBool miss,haszerorows;
PetscErrorCode ierr;
PetscFunctionBeginUser;
if (diag == 0.) {
ierr = PetscStrcpy(diagstr,"zero");CHKERRQ(ierr);
} else {
ierr = PetscStrcpy(diagstr,"nonzero");CHKERRQ(ierr);
}
ierr = ISView(is,NULL);CHKERRQ(ierr);
ierr = MatGetLocalToGlobalMapping(A,&l2gr,&l2gc);CHKERRQ(ierr);
/* tests MatDuplicate and MatCopy */
if (diag == 0.) {
ierr = MatDuplicate(A,MAT_COPY_VALUES,&B);CHKERRQ(ierr);
} else {
ierr = MatDuplicate(A,MAT_DO_NOT_COPY_VALUES,&B);CHKERRQ(ierr);
ierr = MatCopy(A,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
ierr = MatISGetLocalMat(B,&lB);CHKERRQ(ierr);
ierr = MatHasOperation(lB,MATOP_ZERO_ROWS,&haszerorows);CHKERRQ(ierr);
if (squaretest && haszerorows) {
ierr = MatCreateVecs(B,&x,&b);CHKERRQ(ierr);
ierr = MatDuplicate(B,MAT_COPY_VALUES,&B2);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(b,l2gr);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(x,l2gc);CHKERRQ(ierr);
ierr = VecSetRandom(x,NULL);CHKERRQ(ierr);
ierr = VecSetRandom(b,NULL);CHKERRQ(ierr);
/* mimic b[is] = x[is] */
ierr = VecDuplicate(b,&b2);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(b2,l2gr);CHKERRQ(ierr);
ierr = VecCopy(b,b2);CHKERRQ(ierr);
ierr = ISGetLocalSize(is,&n);CHKERRQ(ierr);
ierr = ISGetIndices(is,&idxs);CHKERRQ(ierr);
ierr = VecGetSize(x,&N);CHKERRQ(ierr);
for (i=0;i<n;i++) {
if (0 <= idxs[i] && idxs[i] < N) {
ierr = VecSetValue(b2,idxs[i],diag,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecSetValue(x,idxs[i],1.,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = VecAssemblyBegin(b2);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b2);CHKERRQ(ierr);
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&idxs);CHKERRQ(ierr);
/* test ZeroRows on MATIS */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsIS(B,is,diag,x,b);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRowsColumns (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsColumnsIS(B2,is,diag,NULL,NULL);CHKERRQ(ierr);
} else if (haszerorows) {
/* test ZeroRows on MATIS */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
ierr = MatZeroRowsIS(B,is,diag,NULL,NULL);CHKERRQ(ierr);
b = b2 = x = NULL;
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Skipping MatZeroRows (diag %s)\n",diagstr);CHKERRQ(ierr);
b = b2 = x = NULL;
}
if (squaretest && haszerorows) {
ierr = VecAXPY(b2,-1.,b);CHKERRQ(ierr);
ierr = VecNorm(b2,NORM_INFINITY,&error);CHKERRQ(ierr);
if (error > PETSC_SQRT_MACHINE_EPSILON) SETERRQ2(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"ERROR IN ZEROROWS ON B %g (diag %s)",error,diagstr);
}
/* test MatMissingDiagonal */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Test MatMissingDiagonal\n");CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MatMissingDiagonal(B,&miss,&d);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(B,&rst,&ren);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD, "[%d] [%D,%D) Missing %d, row %D (diag %s)\n",rank,rst,ren,(int)miss,d,diagstr);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&b2);CHKERRQ(ierr);
/* check the result of ZeroRows with that from MPIAIJ routines
assuming that MatConvert_IS_XAIJ and MatZeroRows_MPIAIJ work fine */
if (haszerorows) {
ierr = MatDuplicate(Afull,MAT_COPY_VALUES,&Bcheck);CHKERRQ(ierr);
ierr = MatSetOption(Bcheck,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatZeroRowsIS(Bcheck,is,diag,NULL,NULL);CHKERRQ(ierr);
ierr = CheckMat(B,Bcheck,PETSC_FALSE,"Zerorows");CHKERRQ(ierr);
ierr = MatDestroy(&Bcheck);CHKERRQ(ierr);
}
ierr = MatDestroy(&B);CHKERRQ(ierr);
if (B2) { /* test MatZeroRowsColumns */
ierr = MatDuplicate(Afull,MAT_COPY_VALUES,&B);CHKERRQ(ierr);
ierr = MatSetOption(B,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatZeroRowsColumnsIS(B,is,diag,NULL,NULL);CHKERRQ(ierr);
ierr = CheckMat(B2,B,PETSC_FALSE,"MatZeroRowsColumns");CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatDestroy(&B2);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode ComputeKSPBDDC(DomainData dd,Mat A,KSP *ksp)
{
PetscErrorCode ierr;
KSP temp_ksp;
PC pc;
IS dirichletIS=0,neumannIS=0,*bddc_dofs_splitting;
PetscInt localsize,*xadj=NULL,*adjncy=NULL;
MatNullSpace near_null_space;
PetscFunctionBeginUser;
ierr = KSPCreate(dd.gcomm,&temp_ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(temp_ksp,A,A);CHKERRQ(ierr);
ierr = KSPSetType(temp_ksp,KSPCG);CHKERRQ(ierr);
ierr = KSPGetPC(temp_ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCBDDC);CHKERRQ(ierr);
localsize = dd.xm_l*dd.ym_l*dd.zm_l;
/* BDDC customization */
/* jumping coefficients case */
ierr = PCISSetSubdomainScalingFactor(pc,dd.scalingfactor);CHKERRQ(ierr);
/* Dofs splitting
Simple stride-1 IS
It is not needed since, by default, PCBDDC assumes a stride-1 split */
ierr = PetscMalloc1(1,&bddc_dofs_splitting);CHKERRQ(ierr);
#if 1
ierr = ISCreateStride(PETSC_COMM_WORLD,localsize,0,1,&bddc_dofs_splitting[0]);CHKERRQ(ierr);
ierr = PCBDDCSetDofsSplittingLocal(pc,1,bddc_dofs_splitting);CHKERRQ(ierr);
#else
/* examples for global ordering */
/* each process lists the nodes it owns */
PetscInt sr,er;
ierr = MatGetOwnershipRange(A,&sr,&er);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_WORLD,er-sr,sr,1,&bddc_dofs_splitting[0]);CHKERRQ(ierr);
ierr = PCBDDCSetDofsSplitting(pc,1,bddc_dofs_splitting);CHKERRQ(ierr);
/* Split can be passed in a more general way since any process can list any node */
#endif
ierr = ISDestroy(&bddc_dofs_splitting[0]);CHKERRQ(ierr);
ierr = PetscFree(bddc_dofs_splitting);CHKERRQ(ierr);
/* Primal constraints implemented by using a near null space attached to A -> now it passes in only the constants
(which in practice is not needed since, by default, PCBDDC build the primal space using constants for quadrature formulas */
#if 0
Vec vecs[2];
PetscRandom rctx;
ierr = MatCreateVecs(A,&vecs[0],&vecs[1]);CHKERRQ(ierr);
ierr = PetscRandomCreate(dd.gcomm,&rctx);CHKERRQ(ierr);
ierr = VecSetRandom(vecs[0],rctx);CHKERRQ(ierr);
ierr = VecSetRandom(vecs[1],rctx);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(dd.gcomm,PETSC_TRUE,2,vecs,&near_null_space);CHKERRQ(ierr);
ierr = VecDestroy(&vecs[0]);CHKERRQ(ierr);
ierr = VecDestroy(&vecs[1]);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);
#else
ierr = MatNullSpaceCreate(dd.gcomm,PETSC_TRUE,0,NULL,&near_null_space);CHKERRQ(ierr);
#endif
ierr = MatSetNearNullSpace(A,near_null_space);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&near_null_space);CHKERRQ(ierr);
/* CSR graph of subdomain dofs */
ierr = BuildCSRGraph(dd,&xadj,&adjncy);CHKERRQ(ierr);
ierr = PCBDDCSetLocalAdjacencyGraph(pc,localsize,xadj,adjncy,PETSC_OWN_POINTER);CHKERRQ(ierr);
/* Neumann/Dirichlet indices on the global boundary */
if (dd.DBC_zerorows) {
/* Only in case you eliminate some rows matrix with zerorows function, you need to set dirichlet indices into PCBDDC data */
ierr = ComputeSpecialBoundaryIndices(dd,&dirichletIS,&neumannIS);CHKERRQ(ierr);
ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr);
ierr = PCBDDCSetDirichletBoundariesLocal(pc,dirichletIS);CHKERRQ(ierr);
} else {
if (dd.pure_neumann) {
/* In such a case, all interface nodes lying on the global boundary are neumann nodes */
ierr = ComputeSpecialBoundaryIndices(dd,NULL,&neumannIS);CHKERRQ(ierr);
ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr);
} else {
/* It is wrong setting dirichlet indices without having zeroed the corresponding rows in the global matrix */
/* But we can still compute them since nodes near the dirichlet boundaries does not need to be defined as neumann nodes */
ierr = ComputeSpecialBoundaryIndices(dd,&dirichletIS,&neumannIS);CHKERRQ(ierr);
ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr);
}
}
/* Pass local null space information to local matrices (needed when using approximate local solvers) */
if (dd.ipx || dd.pure_neumann) {
MatNullSpace nsp;
Mat local_mat;
ierr = MatISGetLocalMat(A,&local_mat);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nsp);CHKERRQ(ierr);
ierr = MatSetNullSpace(local_mat,nsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nsp);CHKERRQ(ierr);
}
ierr = KSPSetComputeSingularValues(temp_ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = KSPSetFromOptions(temp_ksp);CHKERRQ(ierr);
ierr = KSPSetUp(temp_ksp);CHKERRQ(ierr);
*ksp = temp_ksp;
ierr = ISDestroy(&dirichletIS);CHKERRQ(ierr);
ierr = ISDestroy(&neumannIS);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
