int main(int argc, char** argv) { PC pc; PetscErrorCode ierr; PetscInt m, nn, M, j, k, ne = 4; PetscReal* coords; Vec x, rhs; Mat A; KSP ksp; PetscMPIInt npe, rank; PetscInitialize(&argc, &argv, NULL, NULL); ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD, &npe); CHKERRQ(ierr); ierr = PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "Linear elasticity in 3D", ""); { char nestring[256]; ierr = PetscSNPrintf(nestring, sizeof nestring, "number of elements in each direction, ne+1 must be a multiple of %D (sizes^{1/3})", (PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5)); ierr = PetscOptionsInt("-ne", nestring, "", ne, &ne, NULL); } ierr = PetscOptionsEnd(); CHKERRQ(ierr); const HpddmOption* const opt = HpddmOptionGet(); { HpddmOptionParse(opt, argc, argv, rank == 0); if (rank) HpddmOptionRemove(opt, "verbosity"); } nn = ne + 1; M = 3 * nn * nn * nn; if (npe == 2) { if (rank == 1) m = 0; else m = nn * nn * nn; npe = 1; } else { m = nn * nn * nn / npe; if (rank == npe - 1) m = nn * nn * nn - (npe - 1) * m; } m *= 3; ierr = KSPCreate(PETSC_COMM_WORLD, &ksp); CHKERRQ(ierr); ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); int i; { PetscInt Istart, Iend, jj, ic; const PetscInt NP = (PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5); const PetscInt ipx = rank % NP, ipy = (rank % (NP * NP)) / NP, ipz = rank / (NP * NP); const PetscInt Ni0 = ipx * (nn / NP), Nj0 = ipy * (nn / NP), Nk0 = ipz * (nn / NP); const PetscInt Ni1 = Ni0 + (m > 0 ? (nn / NP) : 0), Nj1 = Nj0 + (nn / NP), Nk1 = Nk0 + (nn / NP); PetscInt *d_nnz, *o_nnz, osz[4] = {0, 9, 15, 19}, nbc; if (npe != NP * NP * NP) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "npe=%d: npe^{1/3} must be integer", npe); if (nn != NP * (nn / NP)) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "-ne %d: (ne+1)%(npe^{1/3}) must equal zero", ne); ierr = PetscMalloc1(m + 1, &d_nnz); CHKERRQ(ierr); ierr = PetscMalloc1(m + 1, &o_nnz); CHKERRQ(ierr); for (i = Ni0, ic = 0; i < Ni1; i++) { for (j = Nj0; j < Nj1; j++) { for (k = Nk0; k < Nk1; k++) { nbc = 0; if (i == Ni0 || i == Ni1 - 1) nbc++; if (j == Nj0 || j == Nj1 - 1) nbc++; if (k == Nk0 || k == Nk1 - 1) nbc++; for (jj = 0; jj < 3; jj++, ic++) { d_nnz[ic] = 3 * (27 - osz[nbc]); o_nnz[ic] = 3 * osz[nbc]; } } } } if (ic != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "ic %D does not equal m %D", ic, m); ierr = MatCreate(PETSC_COMM_WORLD, &A); CHKERRQ(ierr); ierr = MatSetSizes(A, m, m, M, M); CHKERRQ(ierr); ierr = MatSetBlockSize(A, 3); CHKERRQ(ierr); ierr = MatSetType(A, MATAIJ); CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A, 0, d_nnz); CHKERRQ(ierr); ierr = MatMPIAIJSetPreallocation(A, 0, d_nnz, 0, o_nnz); CHKERRQ(ierr); ierr = PetscFree(d_nnz); CHKERRQ(ierr); ierr = PetscFree(o_nnz); CHKERRQ(ierr); ierr = MatGetOwnershipRange(A, &Istart, &Iend); CHKERRQ(ierr); if (m != Iend - Istart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "m %D does not equal Iend %D - Istart %D", m, Iend, Istart); ierr = VecCreate(PETSC_COMM_WORLD, &x); CHKERRQ(ierr); ierr = VecSetSizes(x, m, M); CHKERRQ(ierr); ierr = VecSetBlockSize(x, 3); CHKERRQ(ierr); ierr = VecSetFromOptions(x); CHKERRQ(ierr); ierr = VecDuplicate(x, &rhs); CHKERRQ(ierr); ierr = PetscMalloc1(m + 1, &coords); CHKERRQ(ierr); coords[m] = -99.0; PetscReal h = 1.0 / ne; for (i = Ni0, ic = 0; i < Ni1; i++) { for (j = Nj0; j < Nj1; j++) { for (k = Nk0; k < Nk1; k++, ic++) { coords[3 * ic] = h * (PetscReal)i; coords[3 * ic + 1] = h * (PetscReal)j; coords[3 * ic + 2] = h * (PetscReal)k; } } } } PetscReal s_r[SIZE_ARRAY_R] = {30, 0.1, 20, 10}; PetscReal x_r[SIZE_ARRAY_R] = {0.5, 0.4, 0.4, 0.4}; PetscReal y_r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4}; PetscReal z_r[SIZE_ARRAY_R] = {0.5, 0.45, 0.4, 0.35}; PetscReal r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4}; AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); MatNullSpace matnull; Vec vec_coords; PetscScalar* c; ierr = VecCreate(MPI_COMM_WORLD, &vec_coords); CHKERRQ(ierr); ierr = VecSetBlockSize(vec_coords, 3); CHKERRQ(ierr); ierr = VecSetSizes(vec_coords, m, PETSC_DECIDE); CHKERRQ(ierr); ierr = VecSetUp(vec_coords); CHKERRQ(ierr); ierr = VecGetArray(vec_coords, &c); CHKERRQ(ierr); for (i = 0; i < m; i++) c[i] = coords[i]; ierr = VecRestoreArray(vec_coords, &c); CHKERRQ(ierr); ierr = MatNullSpaceCreateRigidBody(vec_coords, &matnull); CHKERRQ(ierr); ierr = MatSetNearNullSpace(A, matnull); CHKERRQ(ierr); ierr = MatNullSpaceDestroy(&matnull); CHKERRQ(ierr); ierr = VecDestroy(&vec_coords); CHKERRQ(ierr); ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE); CHKERRQ(ierr); MPI_Barrier(PETSC_COMM_WORLD); double time = MPI_Wtime(); ierr = KSPSetUp(ksp); CHKERRQ(ierr); MPI_Barrier(PETSC_COMM_WORLD); time = MPI_Wtime() - time; ierr = PetscPrintf(PETSC_COMM_WORLD, "--- PC setup = %f\n", time); CHKERRQ(ierr); float t_time[SIZE_ARRAY_R]; int t_its[SIZE_ARRAY_R]; { { ierr = KSPSolve(ksp, rhs, x); CHKERRQ(ierr); ierr = KSPReset(ksp); CHKERRQ(ierr); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); } for (i = 0; i < SIZE_ARRAY_R; ++i) { ierr = VecZeroEntries(x); CHKERRQ(ierr); MPI_Barrier(PETSC_COMM_WORLD); time = MPI_Wtime(); ierr = KSPSolve(ksp, rhs, x); CHKERRQ(ierr); MPI_Barrier(PETSC_COMM_WORLD); t_time[i] = MPI_Wtime() - time; PetscInt its; ierr = KSPGetIterationNumber(ksp, &its); CHKERRQ(ierr); t_its[i] = its; ierr = ComputeError(A, rhs, x); CHKERRQ(ierr); if (i == (SIZE_ARRAY_R - 1)) AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m); else AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); } for (i = 0; i < SIZE_ARRAY_R; ++i) { ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]); CHKERRQ(ierr); if (i > 0) { t_its[0] += t_its[i]; t_time[0] += t_time[i]; } } if (SIZE_ARRAY_R > 1) { ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]); CHKERRQ(ierr); } } { ierr = KSPGetPC(ksp, &pc); CHKERRQ(ierr); HpddmCustomOperator H; H._A = A; H._M = pc; H._mv = mv; H._precond = precond; H._b = rhs; H._x = x; int n; MatGetLocalSize(A, &n, NULL); { ierr = VecZeroEntries(x); K* pt_rhs; K* pt_x; VecGetArray(rhs, &pt_rhs); VecGetArray(x, &pt_x); int previous = HpddmOptionVal(opt, "verbosity"); if (previous > 0) HpddmOptionRemove(opt, "verbosity"); HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD); if (previous > 0) { char buffer[20]; snprintf(buffer, 20, "%d", previous); char* concat = malloc(strlen("-hpddm_verbosity ") + strlen(buffer) + 1); strcpy(concat, "-hpddm_verbosity "); strcat(concat, buffer); HpddmOptionParseString(opt, concat); free(concat); } VecRestoreArray(x, &pt_x); VecRestoreArray(rhs, &pt_rhs); previous = HpddmOptionVal(opt, "krylov_method"); if(previous == 4 || previous == 5) HpddmDestroyRecycling(); ierr = KSPReset(ksp); CHKERRQ(ierr); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); } for (i = 0; i < SIZE_ARRAY_R; ++i) { ierr = VecZeroEntries(x); CHKERRQ(ierr); K* pt_rhs; K* pt_x; VecGetArray(rhs, &pt_rhs); VecGetArray(x, &pt_x); MPI_Barrier(PETSC_COMM_WORLD); time = MPI_Wtime(); t_its[i] = HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD); MPI_Barrier(PETSC_COMM_WORLD); t_time[i] = MPI_Wtime() - time; VecRestoreArray(x, &pt_x); VecRestoreArray(rhs, &pt_rhs); ierr = ComputeError(A, rhs, x); CHKERRQ(ierr); if (i != (SIZE_ARRAY_R - 1)) { AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); } } for (i = 0; i < SIZE_ARRAY_R; ++i) { ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]); CHKERRQ(ierr); if (i > 0) { t_its[0] += t_its[i]; t_time[0] += t_time[i]; } } if (SIZE_ARRAY_R > 1) { ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]); CHKERRQ(ierr); } } ierr = KSPDestroy(&ksp); CHKERRQ(ierr); ierr = VecDestroy(&x); CHKERRQ(ierr); ierr = VecDestroy(&rhs); CHKERRQ(ierr); ierr = MatDestroy(&A); CHKERRQ(ierr); ierr = PetscFree(coords); CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
std::pair<unsigned int, Real> PetscNonlinearSolver<T>::solve (SparseMatrix<T>& jac_in, // System Jacobian Matrix NumericVector<T>& x_in, // Solution vector NumericVector<T>& r_in, // Residual vector const double, // Stopping tolerance const unsigned int) { START_LOG("solve()", "PetscNonlinearSolver"); this->init (); // Make sure the data passed in are really of Petsc types PetscMatrix<T>* jac = libmesh_cast_ptr<PetscMatrix<T>*>(&jac_in); PetscVector<T>* x = libmesh_cast_ptr<PetscVector<T>*>(&x_in); PetscVector<T>* r = libmesh_cast_ptr<PetscVector<T>*>(&r_in); PetscErrorCode ierr=0; PetscInt n_iterations =0; // Should actually be a PetscReal, but I don't know which version of PETSc first introduced PetscReal Real final_residual_norm=0.; ierr = SNESSetFunction (_snes, r->vec(), __libmesh_petsc_snes_residual, this); LIBMESH_CHKERRABORT(ierr); // Only set the jacobian function if we've been provided with something to call. // This allows a user to set their own jacobian function if they want to if (this->jacobian || this->jacobian_object || this->residual_and_jacobian_object) { ierr = SNESSetJacobian (_snes, jac->mat(), jac->mat(), __libmesh_petsc_snes_jacobian, this); LIBMESH_CHKERRABORT(ierr); } #if !PETSC_VERSION_LESS_THAN(3,3,0) // Only set the nullspace if we have a way of computing it and the result is non-empty. if (this->nullspace || this->nullspace_object) { MatNullSpace msp; this->build_mat_null_space(this->nullspace_object, this->nullspace, &msp); if (msp) { ierr = MatSetNullSpace(jac->mat(), msp); LIBMESH_CHKERRABORT(ierr); ierr = MatNullSpaceDestroy(&msp); LIBMESH_CHKERRABORT(ierr); } } // Only set the nearnullspace if we have a way of computing it and the result is non-empty. if (this->nearnullspace || this->nearnullspace_object) { MatNullSpace msp = PETSC_NULL; this->build_mat_null_space(this->nearnullspace_object, this->nearnullspace, &msp); if(msp) { ierr = MatSetNearNullSpace(jac->mat(), msp); LIBMESH_CHKERRABORT(ierr); ierr = MatNullSpaceDestroy(&msp); LIBMESH_CHKERRABORT(ierr); } } #endif // Have the Krylov subspace method use our good initial guess rather than 0 KSP ksp; ierr = SNESGetKSP (_snes, &ksp); LIBMESH_CHKERRABORT(ierr); // Set the tolerances for the iterative solver. Use the user-supplied // tolerance for the relative residual & leave the others at default values ierr = KSPSetTolerances (ksp, this->initial_linear_tolerance, PETSC_DEFAULT, PETSC_DEFAULT, this->max_linear_iterations); LIBMESH_CHKERRABORT(ierr); // Set the tolerances for the non-linear solver. ierr = SNESSetTolerances(_snes, this->absolute_residual_tolerance, this->relative_residual_tolerance, this->relative_step_tolerance, this->max_nonlinear_iterations, this->max_function_evaluations); LIBMESH_CHKERRABORT(ierr); //Pull in command-line options KSPSetFromOptions(ksp); SNESSetFromOptions(_snes); if (this->user_presolve) this->user_presolve(this->system()); //Set the preconditioning matrix if(this->_preconditioner) { this->_preconditioner->set_matrix(jac_in); this->_preconditioner->init(); } // ierr = KSPSetInitialGuessNonzero (ksp, PETSC_TRUE); // LIBMESH_CHKERRABORT(ierr); // Older versions (at least up to 2.1.5) of SNESSolve took 3 arguments, // the last one being a pointer to an int to hold the number of iterations required. # if PETSC_VERSION_LESS_THAN(2,2,0) ierr = SNESSolve (_snes, x->vec(), &n_iterations); LIBMESH_CHKERRABORT(ierr); // 2.2.x style #elif PETSC_VERSION_LESS_THAN(2,3,0) ierr = SNESSolve (_snes, x->vec()); LIBMESH_CHKERRABORT(ierr); // 2.3.x & newer style #else ierr = SNESSolve (_snes, PETSC_NULL, x->vec()); LIBMESH_CHKERRABORT(ierr); ierr = SNESGetIterationNumber(_snes,&n_iterations); LIBMESH_CHKERRABORT(ierr); ierr = SNESGetLinearSolveIterations(_snes, &_n_linear_iterations); LIBMESH_CHKERRABORT(ierr); ierr = SNESGetFunctionNorm(_snes,&final_residual_norm); LIBMESH_CHKERRABORT(ierr); #endif // Get and store the reason for convergence SNESGetConvergedReason(_snes, &_reason); //Based on Petsc 2.3.3 documentation all diverged reasons are negative this->converged = (_reason >= 0); this->clear(); STOP_LOG("solve()", "PetscNonlinearSolver"); // return the # of its. and the final residual norm. return std::make_pair(n_iterations, final_residual_norm); }
static PetscErrorCode ComputeKSPBDDC(DomainData dd,Mat A,KSP *ksp) { PetscErrorCode ierr; KSP temp_ksp; PC pc; IS dirichletIS=0,neumannIS=0,*bddc_dofs_splitting; PetscInt localsize,*xadj=NULL,*adjncy=NULL; MatNullSpace near_null_space; PetscFunctionBeginUser; ierr = KSPCreate(dd.gcomm,&temp_ksp);CHKERRQ(ierr); ierr = KSPSetOperators(temp_ksp,A,A);CHKERRQ(ierr); ierr = KSPSetType(temp_ksp,KSPCG);CHKERRQ(ierr); ierr = KSPGetPC(temp_ksp,&pc);CHKERRQ(ierr); ierr = PCSetType(pc,PCBDDC);CHKERRQ(ierr); localsize = dd.xm_l*dd.ym_l*dd.zm_l; /* BDDC customization */ /* jumping coefficients case */ ierr = PCISSetSubdomainScalingFactor(pc,dd.scalingfactor);CHKERRQ(ierr); /* Dofs splitting Simple stride-1 IS It is not needed since, by default, PCBDDC assumes a stride-1 split */ ierr = PetscMalloc1(1,&bddc_dofs_splitting);CHKERRQ(ierr); #if 1 ierr = ISCreateStride(PETSC_COMM_WORLD,localsize,0,1,&bddc_dofs_splitting[0]);CHKERRQ(ierr); ierr = PCBDDCSetDofsSplittingLocal(pc,1,bddc_dofs_splitting);CHKERRQ(ierr); #else /* examples for global ordering */ /* each process lists the nodes it owns */ PetscInt sr,er; ierr = MatGetOwnershipRange(A,&sr,&er);CHKERRQ(ierr); ierr = ISCreateStride(PETSC_COMM_WORLD,er-sr,sr,1,&bddc_dofs_splitting[0]);CHKERRQ(ierr); ierr = PCBDDCSetDofsSplitting(pc,1,bddc_dofs_splitting);CHKERRQ(ierr); /* Split can be passed in a more general way since any process can list any node */ #endif ierr = ISDestroy(&bddc_dofs_splitting[0]);CHKERRQ(ierr); ierr = PetscFree(bddc_dofs_splitting);CHKERRQ(ierr); /* Primal constraints implemented by using a near null space attached to A -> now it passes in only the constants (which in practice is not needed since, by default, PCBDDC build the primal space using constants for quadrature formulas */ #if 0 Vec vecs[2]; PetscRandom rctx; ierr = MatCreateVecs(A,&vecs[0],&vecs[1]);CHKERRQ(ierr); ierr = PetscRandomCreate(dd.gcomm,&rctx);CHKERRQ(ierr); ierr = VecSetRandom(vecs[0],rctx);CHKERRQ(ierr); ierr = VecSetRandom(vecs[1],rctx);CHKERRQ(ierr); ierr = MatNullSpaceCreate(dd.gcomm,PETSC_TRUE,2,vecs,&near_null_space);CHKERRQ(ierr); ierr = VecDestroy(&vecs[0]);CHKERRQ(ierr); ierr = VecDestroy(&vecs[1]);CHKERRQ(ierr); ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr); #else ierr = MatNullSpaceCreate(dd.gcomm,PETSC_TRUE,0,NULL,&near_null_space);CHKERRQ(ierr); #endif ierr = MatSetNearNullSpace(A,near_null_space);CHKERRQ(ierr); ierr = MatNullSpaceDestroy(&near_null_space);CHKERRQ(ierr); /* CSR graph of subdomain dofs */ ierr = BuildCSRGraph(dd,&xadj,&adjncy);CHKERRQ(ierr); ierr = PCBDDCSetLocalAdjacencyGraph(pc,localsize,xadj,adjncy,PETSC_OWN_POINTER);CHKERRQ(ierr); /* Neumann/Dirichlet indices on the global boundary */ if (dd.DBC_zerorows) { /* Only in case you eliminate some rows matrix with zerorows function, you need to set dirichlet indices into PCBDDC data */ ierr = ComputeSpecialBoundaryIndices(dd,&dirichletIS,&neumannIS);CHKERRQ(ierr); ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr); ierr = PCBDDCSetDirichletBoundariesLocal(pc,dirichletIS);CHKERRQ(ierr); } else { if (dd.pure_neumann) { /* In such a case, all interface nodes lying on the global boundary are neumann nodes */ ierr = ComputeSpecialBoundaryIndices(dd,NULL,&neumannIS);CHKERRQ(ierr); ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr); } else { /* It is wrong setting dirichlet indices without having zeroed the corresponding rows in the global matrix */ /* But we can still compute them since nodes near the dirichlet boundaries does not need to be defined as neumann nodes */ ierr = ComputeSpecialBoundaryIndices(dd,&dirichletIS,&neumannIS);CHKERRQ(ierr); ierr = PCBDDCSetNeumannBoundariesLocal(pc,neumannIS);CHKERRQ(ierr); } } /* Pass local null space information to local matrices (needed when using approximate local solvers) */ if (dd.ipx || dd.pure_neumann) { MatNullSpace nsp; Mat local_mat; ierr = MatISGetLocalMat(A,&local_mat);CHKERRQ(ierr); ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nsp);CHKERRQ(ierr); ierr = MatSetNullSpace(local_mat,nsp);CHKERRQ(ierr); ierr = MatNullSpaceDestroy(&nsp);CHKERRQ(ierr); } ierr = KSPSetComputeSingularValues(temp_ksp,PETSC_TRUE);CHKERRQ(ierr); ierr = KSPSetFromOptions(temp_ksp);CHKERRQ(ierr); ierr = KSPSetUp(temp_ksp);CHKERRQ(ierr); *ksp = temp_ksp; ierr = ISDestroy(&dirichletIS);CHKERRQ(ierr); ierr = ISDestroy(&neumannIS);CHKERRQ(ierr); PetscFunctionReturn(0); }