int main(int argc, char** argv)
{
PC pc;
PetscErrorCode ierr;
PetscInt m, nn, M, j, k, ne = 4;
PetscReal* coords;
Vec x, rhs;
Mat A;
KSP ksp;
PetscMPIInt npe, rank;
PetscInitialize(&argc, &argv, NULL, NULL);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &npe);
CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "Linear elasticity in 3D", "");
{
char nestring[256];
ierr = PetscSNPrintf(nestring, sizeof nestring, "number of elements in each direction, ne+1 must be a multiple of %D (sizes^{1/3})",
(PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5));
ierr = PetscOptionsInt("-ne", nestring, "", ne, &ne, NULL);
}
ierr = PetscOptionsEnd();
CHKERRQ(ierr);
const HpddmOption* const opt = HpddmOptionGet();
{
HpddmOptionParse(opt, argc, argv, rank == 0);
if (rank) HpddmOptionRemove(opt, "verbosity");
}
nn = ne + 1;
M = 3 * nn * nn * nn;
if (npe == 2) {
if (rank == 1)
m = 0;
else
m = nn * nn * nn;
npe = 1;
}
else {
m = nn * nn * nn / npe;
if (rank == npe - 1) m = nn * nn * nn - (npe - 1) * m;
}
m *= 3;
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);
CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);
CHKERRQ(ierr);
int i;
{
PetscInt Istart, Iend, jj, ic;
const PetscInt NP = (PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5);
const PetscInt ipx = rank % NP, ipy = (rank % (NP * NP)) / NP, ipz = rank / (NP * NP);
const PetscInt Ni0 = ipx * (nn / NP), Nj0 = ipy * (nn / NP), Nk0 = ipz * (nn / NP);
const PetscInt Ni1 = Ni0 + (m > 0 ? (nn / NP) : 0), Nj1 = Nj0 + (nn / NP), Nk1 = Nk0 + (nn / NP);
PetscInt *d_nnz, *o_nnz, osz[4] = {0, 9, 15, 19}, nbc;
if (npe != NP * NP * NP) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "npe=%d: npe^{1/3} must be integer", npe);
if (nn != NP * (nn / NP)) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "-ne %d: (ne+1)%(npe^{1/3}) must equal zero", ne);
ierr = PetscMalloc1(m + 1, &d_nnz);
CHKERRQ(ierr);
ierr = PetscMalloc1(m + 1, &o_nnz);
CHKERRQ(ierr);
for (i = Ni0, ic = 0; i < Ni1; i++) {
for (j = Nj0; j < Nj1; j++) {
for (k = Nk0; k < Nk1; k++) {
nbc = 0;
if (i == Ni0 || i == Ni1 - 1) nbc++;
if (j == Nj0 || j == Nj1 - 1) nbc++;
if (k == Nk0 || k == Nk1 - 1) nbc++;
for (jj = 0; jj < 3; jj++, ic++) {
d_nnz[ic] = 3 * (27 - osz[nbc]);
o_nnz[ic] = 3 * osz[nbc];
}
}
}
}
if (ic != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "ic %D does not equal m %D", ic, m);
ierr = MatCreate(PETSC_COMM_WORLD, &A);
CHKERRQ(ierr);
ierr = MatSetSizes(A, m, m, M, M);
CHKERRQ(ierr);
ierr = MatSetBlockSize(A, 3);
CHKERRQ(ierr);
ierr = MatSetType(A, MATAIJ);
CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A, 0, d_nnz);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A, 0, d_nnz, 0, o_nnz);
CHKERRQ(ierr);
ierr = PetscFree(d_nnz);
CHKERRQ(ierr);
ierr = PetscFree(o_nnz);
CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A, &Istart, &Iend);
CHKERRQ(ierr);
if (m != Iend - Istart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "m %D does not equal Iend %D - Istart %D", m, Iend, Istart);
ierr = VecCreate(PETSC_COMM_WORLD, &x);
CHKERRQ(ierr);
ierr = VecSetSizes(x, m, M);
CHKERRQ(ierr);
ierr = VecSetBlockSize(x, 3);
CHKERRQ(ierr);
ierr = VecSetFromOptions(x);
CHKERRQ(ierr);
ierr = VecDuplicate(x, &rhs);
CHKERRQ(ierr);
ierr = PetscMalloc1(m + 1, &coords);
CHKERRQ(ierr);
coords[m] = -99.0;
PetscReal h = 1.0 / ne;
for (i = Ni0, ic = 0; i < Ni1; i++) {
for (j = Nj0; j < Nj1; j++) {
for (k = Nk0; k < Nk1; k++, ic++) {
coords[3 * ic] = h * (PetscReal)i;
coords[3 * ic + 1] = h * (PetscReal)j;
coords[3 * ic + 2] = h * (PetscReal)k;
}
}
}
}
PetscReal s_r[SIZE_ARRAY_R] = {30, 0.1, 20, 10};
PetscReal x_r[SIZE_ARRAY_R] = {0.5, 0.4, 0.4, 0.4};
PetscReal y_r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4};
PetscReal z_r[SIZE_ARRAY_R] = {0.5, 0.45, 0.4, 0.35};
PetscReal r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4};
AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
MatNullSpace matnull;
Vec vec_coords;
PetscScalar* c;
ierr = VecCreate(MPI_COMM_WORLD, &vec_coords);
CHKERRQ(ierr);
ierr = VecSetBlockSize(vec_coords, 3);
CHKERRQ(ierr);
ierr = VecSetSizes(vec_coords, m, PETSC_DECIDE);
CHKERRQ(ierr);
ierr = VecSetUp(vec_coords);
CHKERRQ(ierr);
ierr = VecGetArray(vec_coords, &c);
CHKERRQ(ierr);
for (i = 0; i < m; i++) c[i] = coords[i];
ierr = VecRestoreArray(vec_coords, &c);
CHKERRQ(ierr);
ierr = MatNullSpaceCreateRigidBody(vec_coords, &matnull);
CHKERRQ(ierr);
ierr = MatSetNearNullSpace(A, matnull);
CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&matnull);
CHKERRQ(ierr);
ierr = VecDestroy(&vec_coords);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
double time = MPI_Wtime();
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime() - time;
ierr = PetscPrintf(PETSC_COMM_WORLD, "--- PC setup = %f\n", time);
CHKERRQ(ierr);
float t_time[SIZE_ARRAY_R];
int t_its[SIZE_ARRAY_R];
{
{
ierr = KSPSolve(ksp, rhs, x);
CHKERRQ(ierr);
ierr = KSPReset(ksp);
CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = VecZeroEntries(x);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime();
ierr = KSPSolve(ksp, rhs, x);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
t_time[i] = MPI_Wtime() - time;
PetscInt its;
ierr = KSPGetIterationNumber(ksp, &its);
CHKERRQ(ierr);
t_its[i] = its;
ierr = ComputeError(A, rhs, x);
CHKERRQ(ierr);
if (i == (SIZE_ARRAY_R - 1))
AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m);
else
AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]);
CHKERRQ(ierr);
if (i > 0) {
t_its[0] += t_its[i];
t_time[0] += t_time[i];
}
}
if (SIZE_ARRAY_R > 1) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]);
CHKERRQ(ierr);
}
}
{
ierr = KSPGetPC(ksp, &pc);
CHKERRQ(ierr);
HpddmCustomOperator H;
H._A = A;
H._M = pc;
H._mv = mv;
H._precond = precond;
H._b = rhs;
H._x = x;
int n;
MatGetLocalSize(A, &n, NULL);
{
ierr = VecZeroEntries(x);
K* pt_rhs;
K* pt_x;
VecGetArray(rhs, &pt_rhs);
VecGetArray(x, &pt_x);
int previous = HpddmOptionVal(opt, "verbosity");
if (previous > 0) HpddmOptionRemove(opt, "verbosity");
HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD);
if (previous > 0) {
char buffer[20];
snprintf(buffer, 20, "%d", previous);
char* concat = malloc(strlen("-hpddm_verbosity ") + strlen(buffer) + 1);
strcpy(concat, "-hpddm_verbosity ");
strcat(concat, buffer);
HpddmOptionParseString(opt, concat);
free(concat);
}
VecRestoreArray(x, &pt_x);
VecRestoreArray(rhs, &pt_rhs);
previous = HpddmOptionVal(opt, "krylov_method");
if(previous == 4 || previous == 5) HpddmDestroyRecycling();
ierr = KSPReset(ksp);
CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = VecZeroEntries(x);
CHKERRQ(ierr);
K* pt_rhs;
K* pt_x;
VecGetArray(rhs, &pt_rhs);
VecGetArray(x, &pt_x);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime();
t_its[i] = HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD);
MPI_Barrier(PETSC_COMM_WORLD);
t_time[i] = MPI_Wtime() - time;
VecRestoreArray(x, &pt_x);
VecRestoreArray(rhs, &pt_rhs);
ierr = ComputeError(A, rhs, x);
CHKERRQ(ierr);
if (i != (SIZE_ARRAY_R - 1)) {
AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]);
CHKERRQ(ierr);
if (i > 0) {
t_its[0] += t_its[i];
t_time[0] += t_time[i];
}
}
if (SIZE_ARRAY_R > 1) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]);
CHKERRQ(ierr);
}
}
ierr = KSPDestroy(&ksp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&rhs);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = PetscFree(coords);
CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
std::pair<unsigned int, Real>
PetscNonlinearSolver<T>::solve (SparseMatrix<T>& jac_in, // System Jacobian Matrix
NumericVector<T>& x_in, // Solution vector
NumericVector<T>& r_in, // Residual vector
const double, // Stopping tolerance
const unsigned int)
{
START_LOG("solve()", "PetscNonlinearSolver");
this->init ();
// Make sure the data passed in are really of Petsc types
PetscMatrix<T>* jac = libmesh_cast_ptr<PetscMatrix<T>*>(&jac_in);
PetscVector<T>* x = libmesh_cast_ptr<PetscVector<T>*>(&x_in);
PetscVector<T>* r = libmesh_cast_ptr<PetscVector<T>*>(&r_in);
PetscErrorCode ierr=0;
PetscInt n_iterations =0;
// Should actually be a PetscReal, but I don't know which version of PETSc first introduced PetscReal
Real final_residual_norm=0.;
ierr = SNESSetFunction (_snes, r->vec(), __libmesh_petsc_snes_residual, this);
LIBMESH_CHKERRABORT(ierr);
// Only set the jacobian function if we've been provided with something to call.
// This allows a user to set their own jacobian function if they want to
if (this->jacobian || this->jacobian_object || this->residual_and_jacobian_object)
{
ierr = SNESSetJacobian (_snes, jac->mat(), jac->mat(), __libmesh_petsc_snes_jacobian, this);
LIBMESH_CHKERRABORT(ierr);
}
#if !PETSC_VERSION_LESS_THAN(3,3,0)
// Only set the nullspace if we have a way of computing it and the result is non-empty.
if (this->nullspace || this->nullspace_object)
{
MatNullSpace msp;
this->build_mat_null_space(this->nullspace_object, this->nullspace, &msp);
if (msp)
{
ierr = MatSetNullSpace(jac->mat(), msp);
LIBMESH_CHKERRABORT(ierr);
ierr = MatNullSpaceDestroy(&msp);
LIBMESH_CHKERRABORT(ierr);
}
}
// Only set the nearnullspace if we have a way of computing it and the result is non-empty.
if (this->nearnullspace || this->nearnullspace_object)
{
MatNullSpace msp = PETSC_NULL;
this->build_mat_null_space(this->nearnullspace_object, this->nearnullspace, &msp);
if(msp) {
ierr = MatSetNearNullSpace(jac->mat(), msp);
LIBMESH_CHKERRABORT(ierr);
ierr = MatNullSpaceDestroy(&msp);
LIBMESH_CHKERRABORT(ierr);
}
}
#endif
// Have the Krylov subspace method use our good initial guess rather than 0
KSP ksp;
ierr = SNESGetKSP (_snes, &ksp);
LIBMESH_CHKERRABORT(ierr);
// Set the tolerances for the iterative solver. Use the user-supplied
// tolerance for the relative residual & leave the others at default values
ierr = KSPSetTolerances (ksp, this->initial_linear_tolerance, PETSC_DEFAULT,
PETSC_DEFAULT, this->max_linear_iterations);
LIBMESH_CHKERRABORT(ierr);
// Set the tolerances for the non-linear solver.
ierr = SNESSetTolerances(_snes, this->absolute_residual_tolerance, this->relative_residual_tolerance,
this->relative_step_tolerance, this->max_nonlinear_iterations, this->max_function_evaluations);
LIBMESH_CHKERRABORT(ierr);
//Pull in command-line options
KSPSetFromOptions(ksp);
SNESSetFromOptions(_snes);
if (this->user_presolve)
this->user_presolve(this->system());
//Set the preconditioning matrix
if(this->_preconditioner)
{
this->_preconditioner->set_matrix(jac_in);
this->_preconditioner->init();
}
// ierr = KSPSetInitialGuessNonzero (ksp, PETSC_TRUE);
// LIBMESH_CHKERRABORT(ierr);
// Older versions (at least up to 2.1.5) of SNESSolve took 3 arguments,
// the last one being a pointer to an int to hold the number of iterations required.
# if PETSC_VERSION_LESS_THAN(2,2,0)
ierr = SNESSolve (_snes, x->vec(), &n_iterations);
LIBMESH_CHKERRABORT(ierr);
// 2.2.x style
#elif PETSC_VERSION_LESS_THAN(2,3,0)
ierr = SNESSolve (_snes, x->vec());
LIBMESH_CHKERRABORT(ierr);
// 2.3.x & newer style
#else
ierr = SNESSolve (_snes, PETSC_NULL, x->vec());
LIBMESH_CHKERRABORT(ierr);
ierr = SNESGetIterationNumber(_snes,&n_iterations);
LIBMESH_CHKERRABORT(ierr);
ierr = SNESGetLinearSolveIterations(_snes, &_n_linear_iterations);
LIBMESH_CHKERRABORT(ierr);
ierr = SNESGetFunctionNorm(_snes,&final_residual_norm);
LIBMESH_CHKERRABORT(ierr);
#endif
// Get and store the reason for convergence
SNESGetConvergedReason(_snes, &_reason);
//Based on Petsc 2.3.3 documentation all diverged reasons are negative
this->converged = (_reason >= 0);
this->clear();
STOP_LOG("solve()", "PetscNonlinearSolver");
// return the # of its. and the final residual norm.
return std::make_pair(n_iterations, final_residual_norm);
}
static PetscErrorCode ComputeKSPBDDC(DomainData dd,Mat A,KSP *ksp)
{
PetscErrorCode ierr;
KSP temp_ksp;
PC pc;
IS dirichletIS=0,neumannIS=0,*bddc_dofs_splitting;
PetscInt localsize,*xadj=NULL,*adjncy=NULL;
MatNullSpace near_null_space;
PetscFunctionBeginUser;
ierr = KSPCreate(dd.gcomm,&temp_ksp);CHKERRQ(ierr);
ierr = KSPSetOperators(temp_ksp,A,A);CHKERRQ(ierr);
ierr = KSPSetType(temp_ksp,KSPCG);CHKERRQ(ierr);
ierr = KSPGetPC(temp_ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCBDDC);CHKERRQ(ierr);
localsize = dd.xm_l*dd.ym_l*dd.zm_l;
/* BDDC customization */
/* jumping coefficients case */
ierr = PCISSetSubdomainScalingFactor(pc,dd.scalingfactor);CHKERRQ(ierr);
/* Dofs splitting
Simple stride-1 IS
It is not needed since, by default, PCBDDC assumes a stride-1 split */
ierr = PetscMalloc1(1,&bddc_dofs_splitting);CHKERRQ(ierr);
#if 1
ierr = ISCreateStride(PETSC_COMM_WORLD,localsize,0,1,&bddc_dofs_splitting[0]);CHKERRQ(ierr);
ierr = PCBDDCSetDofsSplittingLocal(pc,1,bddc_dofs_splitting);CHKERRQ(ierr);
#else
/* examples for global ordering */
/* each process lists the nodes it owns */
PetscInt sr,er;
ierr = MatGetOwnershipRange(A,&sr,&er);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_WORLD,er-sr,sr,1,&bddc_dofs_splitting[0]);CHKERRQ(ierr);
ierr = PCBDDCSetDofsSplitting(pc,1,bddc_dofs_splitting);CHKERRQ(ierr);
/* Split can be passed in a more general way since any process can list any node */
#endif
ierr = ISDestroy(&bddc_dofs_splitting[0]);CHKERRQ(ierr);
ierr = PetscFree(bddc_dofs_splitting);CHKERRQ(ierr);
/* Primal constraints implemented by using a near null space attached to A -> now it passes in only the constants
(which in practice is not needed since, by default, PCBDDC build the primal space using constants for quadrature formulas */
#if 0
Vec vecs[2];
PetscRandom rctx;
ierr = MatCreateVecs(A,&vecs[0],&vecs[1]);CHKERRQ(ierr);
ierr = PetscRandomCreate(dd.gcomm,&rctx);CHKERRQ(ierr);
ierr = VecSetRandom(vecs[0],rctx);CHKERRQ(ierr);
ierr = VecSetRandom(vecs[1],rctx);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(dd.gcomm,PETSC_TRUE,2,vecs,&near_null_space);CHKERRQ(ierr);
ierr = VecDestroy(&vecs[0]);CHKERRQ(ierr);
ierr = VecDestroy(&vecs[1]);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);
#else
ierr = MatNullSpaceCreate(dd.gcomm,PETSC_TRUE,0,NULL,&near_null_space);CHKERRQ(ierr);
#endif
ierr = MatSetNearNullSpace(A,near_null_space);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&near_null_space);CHKERRQ(ierr);
/* CSR graph of subdomain dofs */
ierr = BuildCSRGraph(dd,&xadj,&adjncy);CHKERRQ(ierr);
ierr = PCBDDCSetLocalAdjacencyGraph(pc,localsize,xadj,adjncy,PETSC_OWN_POINTER);CHKERRQ(ierr);
/* Neumann/Dirichlet indices on the global boundary */
if (dd.DBC_zerorows) {
/* Only in case you eliminate some rows matrix with zerorows function, you need to set dirichlet indices into PCBDDC data */
ierr = ComputeSpecialBoundaryIndices(dd,&dirichletIS,&neumannIS);CHKERRQ(ierr);
ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr);
ierr = PCBDDCSetDirichletBoundariesLocal(pc,dirichletIS);CHKERRQ(ierr);
} else {
if (dd.pure_neumann) {
/* In such a case, all interface nodes lying on the global boundary are neumann nodes */
ierr = ComputeSpecialBoundaryIndices(dd,NULL,&neumannIS);CHKERRQ(ierr);
ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr);
} else {
/* It is wrong setting dirichlet indices without having zeroed the corresponding rows in the global matrix */
/* But we can still compute them since nodes near the dirichlet boundaries does not need to be defined as neumann nodes */
ierr = ComputeSpecialBoundaryIndices(dd,&dirichletIS,&neumannIS);CHKERRQ(ierr);
ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr);
}
}
/* Pass local null space information to local matrices (needed when using approximate local solvers) */
if (dd.ipx || dd.pure_neumann) {
MatNullSpace nsp;
Mat local_mat;
ierr = MatISGetLocalMat(A,&local_mat);CHKERRQ(ierr);
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nsp);CHKERRQ(ierr);
ierr = MatSetNullSpace(local_mat,nsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nsp);CHKERRQ(ierr);
}
ierr = KSPSetComputeSingularValues(temp_ksp,PETSC_TRUE);CHKERRQ(ierr);
ierr = KSPSetFromOptions(temp_ksp);CHKERRQ(ierr);
ierr = KSPSetUp(temp_ksp);CHKERRQ(ierr);
*ksp = temp_ksp;
ierr = ISDestroy(&dirichletIS);CHKERRQ(ierr);
ierr = ISDestroy(&neumannIS);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
