PetscVector<Number> assemble_coupled_rhs (EquationSystems& es,Tree& tree, Mesh& mesh) { std::cout<< "Assembling big rhs " <<std::endl; TransientLinearImplicitSystem & system = es.add_system<TransientLinearImplicitSystem> ("Newton-update"); NumericVector< Number > &solution_in= *(system.solution); NumericVector< Number > &rhs_in = *(system.rhs); Number size_mat = system.rhs->size(); Real size_t=tree.number_nodes+tree.number_edges; //Sort out rhs - put into big vector Vec x; VecCreate(PETSC_COMM_WORLD,&x); PetscObjectSetName((PetscObject) x, "Solution"); VecSetSizes(x,PETSC_DECIDE,size_mat); VecSetFromOptions(x); PetscVector<Number> xp(x) ; xp.add(1,solution_in); Vec r; VecCreate(PETSC_COMM_WORLD,&r); PetscObjectSetName((PetscObject) r, "rhs"); VecSetSizes(r,PETSC_DECIDE,size_mat); VecSetFromOptions(r); PetscVector<Number> rp(r) ; rp.add(1,rhs_in); Vec rt; rt=tree.make_tree_rhs( ); PetscVector<Number> rtp(rt) ; Vec big_r; VecCreate(PETSC_COMM_WORLD,&big_r); PetscObjectSetName((PetscObject) big_r, "big_rhs"); VecSetSizes(big_r,PETSC_DECIDE,size_mat+size_t); //VecSetSizes(big_r,PETSC_DECIDE,size_mat); VecSetFromOptions(big_r); // add system.rhs into big_rp for (int i=0; i<size_mat; i++) { //if(abs(rhs_in(i))>0){ VecSetValue(big_r,i,rp(i),INSERT_VALUES); //} } // add extra tree rhs entries should all be 0 if p0=0 for (int i=size_mat; i<size_mat+size_t; i++) { VecSetValue(big_r, i , rtp(i-size_mat) ,INSERT_VALUES); } PetscVector<Number> big_rp(big_r) ; return big_rp; }
int main(int argc, char** argv) { PC pc; PetscErrorCode ierr; PetscInt m, nn, M, j, k, ne = 4; PetscReal* coords; Vec x, rhs; Mat A; KSP ksp; PetscMPIInt npe, rank; PetscInitialize(&argc, &argv, NULL, NULL); ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank); CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD, &npe); CHKERRQ(ierr); ierr = PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "Linear elasticity in 3D", ""); { char nestring[256]; ierr = PetscSNPrintf(nestring, sizeof nestring, "number of elements in each direction, ne+1 must be a multiple of %D (sizes^{1/3})", (PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5)); ierr = PetscOptionsInt("-ne", nestring, "", ne, &ne, NULL); } ierr = PetscOptionsEnd(); CHKERRQ(ierr); const HpddmOption* const opt = HpddmOptionGet(); { HpddmOptionParse(opt, argc, argv, rank == 0); if (rank) HpddmOptionRemove(opt, "verbosity"); } nn = ne + 1; M = 3 * nn * nn * nn; if (npe == 2) { if (rank == 1) m = 0; else m = nn * nn * nn; npe = 1; } else { m = nn * nn * nn / npe; if (rank == npe - 1) m = nn * nn * nn - (npe - 1) * m; } m *= 3; ierr = KSPCreate(PETSC_COMM_WORLD, &ksp); CHKERRQ(ierr); ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); int i; { PetscInt Istart, Iend, jj, ic; const PetscInt NP = (PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5); const PetscInt ipx = rank % NP, ipy = (rank % (NP * NP)) / NP, ipz = rank / (NP * NP); const PetscInt Ni0 = ipx * (nn / NP), Nj0 = ipy * (nn / NP), Nk0 = ipz * (nn / NP); const PetscInt Ni1 = Ni0 + (m > 0 ? (nn / NP) : 0), Nj1 = Nj0 + (nn / NP), Nk1 = Nk0 + (nn / NP); PetscInt *d_nnz, *o_nnz, osz[4] = {0, 9, 15, 19}, nbc; if (npe != NP * NP * NP) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "npe=%d: npe^{1/3} must be integer", npe); if (nn != NP * (nn / NP)) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "-ne %d: (ne+1)%(npe^{1/3}) must equal zero", ne); ierr = PetscMalloc1(m + 1, &d_nnz); CHKERRQ(ierr); ierr = PetscMalloc1(m + 1, &o_nnz); CHKERRQ(ierr); for (i = Ni0, ic = 0; i < Ni1; i++) { for (j = Nj0; j < Nj1; j++) { for (k = Nk0; k < Nk1; k++) { nbc = 0; if (i == Ni0 || i == Ni1 - 1) nbc++; if (j == Nj0 || j == Nj1 - 1) nbc++; if (k == Nk0 || k == Nk1 - 1) nbc++; for (jj = 0; jj < 3; jj++, ic++) { d_nnz[ic] = 3 * (27 - osz[nbc]); o_nnz[ic] = 3 * osz[nbc]; } } } } if (ic != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "ic %D does not equal m %D", ic, m); ierr = MatCreate(PETSC_COMM_WORLD, &A); CHKERRQ(ierr); ierr = MatSetSizes(A, m, m, M, M); CHKERRQ(ierr); ierr = MatSetBlockSize(A, 3); CHKERRQ(ierr); ierr = MatSetType(A, MATAIJ); CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(A, 0, d_nnz); CHKERRQ(ierr); ierr = MatMPIAIJSetPreallocation(A, 0, d_nnz, 0, o_nnz); CHKERRQ(ierr); ierr = PetscFree(d_nnz); CHKERRQ(ierr); ierr = PetscFree(o_nnz); CHKERRQ(ierr); ierr = MatGetOwnershipRange(A, &Istart, &Iend); CHKERRQ(ierr); if (m != Iend - Istart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "m %D does not equal Iend %D - Istart %D", m, Iend, Istart); ierr = VecCreate(PETSC_COMM_WORLD, &x); CHKERRQ(ierr); ierr = VecSetSizes(x, m, M); CHKERRQ(ierr); ierr = VecSetBlockSize(x, 3); CHKERRQ(ierr); ierr = VecSetFromOptions(x); CHKERRQ(ierr); ierr = VecDuplicate(x, &rhs); CHKERRQ(ierr); ierr = PetscMalloc1(m + 1, &coords); CHKERRQ(ierr); coords[m] = -99.0; PetscReal h = 1.0 / ne; for (i = Ni0, ic = 0; i < Ni1; i++) { for (j = Nj0; j < Nj1; j++) { for (k = Nk0; k < Nk1; k++, ic++) { coords[3 * ic] = h * (PetscReal)i; coords[3 * ic + 1] = h * (PetscReal)j; coords[3 * ic + 2] = h * (PetscReal)k; } } } } PetscReal s_r[SIZE_ARRAY_R] = {30, 0.1, 20, 10}; PetscReal x_r[SIZE_ARRAY_R] = {0.5, 0.4, 0.4, 0.4}; PetscReal y_r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4}; PetscReal z_r[SIZE_ARRAY_R] = {0.5, 0.45, 0.4, 0.35}; PetscReal r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4}; AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); MatNullSpace matnull; Vec vec_coords; PetscScalar* c; ierr = VecCreate(MPI_COMM_WORLD, &vec_coords); CHKERRQ(ierr); ierr = VecSetBlockSize(vec_coords, 3); CHKERRQ(ierr); ierr = VecSetSizes(vec_coords, m, PETSC_DECIDE); CHKERRQ(ierr); ierr = VecSetUp(vec_coords); CHKERRQ(ierr); ierr = VecGetArray(vec_coords, &c); CHKERRQ(ierr); for (i = 0; i < m; i++) c[i] = coords[i]; ierr = VecRestoreArray(vec_coords, &c); CHKERRQ(ierr); ierr = MatNullSpaceCreateRigidBody(vec_coords, &matnull); CHKERRQ(ierr); ierr = MatSetNearNullSpace(A, matnull); CHKERRQ(ierr); ierr = MatNullSpaceDestroy(&matnull); CHKERRQ(ierr); ierr = VecDestroy(&vec_coords); CHKERRQ(ierr); ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE); CHKERRQ(ierr); MPI_Barrier(PETSC_COMM_WORLD); double time = MPI_Wtime(); ierr = KSPSetUp(ksp); CHKERRQ(ierr); MPI_Barrier(PETSC_COMM_WORLD); time = MPI_Wtime() - time; ierr = PetscPrintf(PETSC_COMM_WORLD, "--- PC setup = %f\n", time); CHKERRQ(ierr); float t_time[SIZE_ARRAY_R]; int t_its[SIZE_ARRAY_R]; { { ierr = KSPSolve(ksp, rhs, x); CHKERRQ(ierr); ierr = KSPReset(ksp); CHKERRQ(ierr); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); } for (i = 0; i < SIZE_ARRAY_R; ++i) { ierr = VecZeroEntries(x); CHKERRQ(ierr); MPI_Barrier(PETSC_COMM_WORLD); time = MPI_Wtime(); ierr = KSPSolve(ksp, rhs, x); CHKERRQ(ierr); MPI_Barrier(PETSC_COMM_WORLD); t_time[i] = MPI_Wtime() - time; PetscInt its; ierr = KSPGetIterationNumber(ksp, &its); CHKERRQ(ierr); t_its[i] = its; ierr = ComputeError(A, rhs, x); CHKERRQ(ierr); if (i == (SIZE_ARRAY_R - 1)) AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m); else AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); } for (i = 0; i < SIZE_ARRAY_R; ++i) { ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]); CHKERRQ(ierr); if (i > 0) { t_its[0] += t_its[i]; t_time[0] += t_time[i]; } } if (SIZE_ARRAY_R > 1) { ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]); CHKERRQ(ierr); } } { ierr = KSPGetPC(ksp, &pc); CHKERRQ(ierr); HpddmCustomOperator H; H._A = A; H._M = pc; H._mv = mv; H._precond = precond; H._b = rhs; H._x = x; int n; MatGetLocalSize(A, &n, NULL); { ierr = VecZeroEntries(x); K* pt_rhs; K* pt_x; VecGetArray(rhs, &pt_rhs); VecGetArray(x, &pt_x); int previous = HpddmOptionVal(opt, "verbosity"); if (previous > 0) HpddmOptionRemove(opt, "verbosity"); HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD); if (previous > 0) { char buffer[20]; snprintf(buffer, 20, "%d", previous); char* concat = malloc(strlen("-hpddm_verbosity ") + strlen(buffer) + 1); strcpy(concat, "-hpddm_verbosity "); strcat(concat, buffer); HpddmOptionParseString(opt, concat); free(concat); } VecRestoreArray(x, &pt_x); VecRestoreArray(rhs, &pt_rhs); previous = HpddmOptionVal(opt, "krylov_method"); if(previous == 4 || previous == 5) HpddmDestroyRecycling(); ierr = KSPReset(ksp); CHKERRQ(ierr); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); } for (i = 0; i < SIZE_ARRAY_R; ++i) { ierr = VecZeroEntries(x); CHKERRQ(ierr); K* pt_rhs; K* pt_x; VecGetArray(rhs, &pt_rhs); VecGetArray(x, &pt_x); MPI_Barrier(PETSC_COMM_WORLD); time = MPI_Wtime(); t_its[i] = HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD); MPI_Barrier(PETSC_COMM_WORLD); t_time[i] = MPI_Wtime() - time; VecRestoreArray(x, &pt_x); VecRestoreArray(rhs, &pt_rhs); ierr = ComputeError(A, rhs, x); CHKERRQ(ierr); if (i != (SIZE_ARRAY_R - 1)) { AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m); ierr = KSPSetOperators(ksp, A, A); CHKERRQ(ierr); ierr = KSPSetUp(ksp); CHKERRQ(ierr); } } for (i = 0; i < SIZE_ARRAY_R; ++i) { ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]); CHKERRQ(ierr); if (i > 0) { t_its[0] += t_its[i]; t_time[0] += t_time[i]; } } if (SIZE_ARRAY_R > 1) { ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]); CHKERRQ(ierr); } } ierr = KSPDestroy(&ksp); CHKERRQ(ierr); ierr = VecDestroy(&x); CHKERRQ(ierr); ierr = VecDestroy(&rhs); CHKERRQ(ierr); ierr = MatDestroy(&A); CHKERRQ(ierr); ierr = PetscFree(coords); CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
For MATSEQDENSE matrix, the factorization is just a thin wrapper to LAPACK \n\n"; #include <petscmat.h> #undef __FUNCT__ #define __FUNCT__ "main" int main(int argc,char **argv) { Mat mat,F,RHS,SOLU; MatInfo info; PetscErrorCode ierr; PetscInt m = 10,n = 10,i,j,rstart,rend,nrhs=2; PetscScalar value = 1.0; Vec x,y,b,ytmp; PetscReal norm,tol=1.e-15; PetscMPIInt size; PetscScalar *rhs_array,*solu_array; PetscRandom rand; PetscScalar *array,rval; PetscInitialize(&argc,&argv,(char*) 0,help); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"This is a uniprocessor example only!"); /* create single vectors */ ierr = VecCreate(PETSC_COMM_WORLD,&y);CHKERRQ(ierr); ierr = VecSetSizes(y,PETSC_DECIDE,m);CHKERRQ(ierr); ierr = VecSetFromOptions(y);CHKERRQ(ierr); ierr = VecDuplicate(y,&x);CHKERRQ(ierr); ierr = VecDuplicate(y,&ytmp);CHKERRQ(ierr); ierr = VecSet(x,value);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr); ierr = VecSetSizes(b,PETSC_DECIDE,n);CHKERRQ(ierr); ierr = VecSetFromOptions(b);CHKERRQ(ierr); /* create multiple vectors RHS and SOLU */ ierr = MatCreate(PETSC_COMM_WORLD,&RHS);CHKERRQ(ierr); ierr = MatSetSizes(RHS,PETSC_DECIDE,PETSC_DECIDE,n,nrhs);CHKERRQ(ierr); ierr = MatSetType(RHS,MATDENSE);CHKERRQ(ierr); ierr = MatSetFromOptions(RHS);CHKERRQ(ierr); ierr = MatSeqDenseSetPreallocation(RHS,NULL);CHKERRQ(ierr); ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr); ierr = MatDenseGetArray(RHS,&array);CHKERRQ(ierr); for (j=0; j<nrhs; j++) { for (i=0; i<n; i++) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); array[n*j+i] = rval; } } ierr = MatDenseRestoreArray(RHS,&array);CHKERRQ(ierr); ierr = MatDuplicate(RHS,MAT_DO_NOT_COPY_VALUES,&SOLU);CHKERRQ(ierr); /* create matrix */ ierr = MatCreateSeqDense(PETSC_COMM_WORLD,m,n,NULL,&mat);CHKERRQ(ierr); ierr = MatGetOwnershipRange(mat,&rstart,&rend);CHKERRQ(ierr); for (i=rstart; i<rend; i++) { value = (PetscReal)i+1; ierr = MatSetValues(mat,1,&i,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr); } ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatGetInfo(mat,MAT_LOCAL,&info);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"matrix nonzeros = %D, allocated nonzeros = %D\n", (PetscInt)info.nz_used,(PetscInt)info.nz_allocated);CHKERRQ(ierr); /* Cholesky factorization - perm and factinfo are ignored by LAPACK */ /* in-place Cholesky */ ierr = MatMult(mat,x,b);CHKERRQ(ierr); ierr = MatDuplicate(mat,MAT_COPY_VALUES,&F);CHKERRQ(ierr); ierr = MatCholeskyFactor(F,0,0);CHKERRQ(ierr); ierr = MatSolve(F,b,y);CHKERRQ(ierr); ierr = MatDestroy(&F);CHKERRQ(ierr); value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr); ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for Cholesky %G\n",norm);CHKERRQ(ierr); } /* out-place Cholesky */ ierr = MatGetFactor(mat,MATSOLVERPETSC,MAT_FACTOR_CHOLESKY,&F);CHKERRQ(ierr); ierr = MatCholeskyFactorSymbolic(F,mat,0,0);CHKERRQ(ierr); ierr = MatCholeskyFactorNumeric(F,mat,0);CHKERRQ(ierr); ierr = MatSolve(F,b,y);CHKERRQ(ierr); value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr); ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for Cholesky %G\n",norm);CHKERRQ(ierr); } ierr = MatDestroy(&F);CHKERRQ(ierr); /* LU factorization - perms and factinfo are ignored by LAPACK */ i = m-1; value = 1.0; ierr = MatSetValues(mat,1,&i,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatMult(mat,x,b);CHKERRQ(ierr); ierr = MatDuplicate(mat,MAT_COPY_VALUES,&F);CHKERRQ(ierr); /* in-place LU */ ierr = MatLUFactor(F,0,0,0);CHKERRQ(ierr); ierr = MatSolve(F,b,y);CHKERRQ(ierr); value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr); ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for LU %G\n",norm);CHKERRQ(ierr); } ierr = MatMatSolve(F,RHS,SOLU);CHKERRQ(ierr); ierr = MatDenseGetArray(SOLU,&solu_array);CHKERRQ(ierr); ierr = MatDenseGetArray(RHS,&rhs_array);CHKERRQ(ierr); for (j=0; j<nrhs; j++) { ierr = VecPlaceArray(y,solu_array+j*m);CHKERRQ(ierr); ierr = VecPlaceArray(b,rhs_array+j*m);CHKERRQ(ierr); ierr = MatMult(mat,y,ytmp);CHKERRQ(ierr); ierr = VecAXPY(ytmp,-1.0,b);CHKERRQ(ierr); /* ytmp = mat*SOLU[:,j] - RHS[:,j] */ ierr = VecNorm(ytmp,NORM_2,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: Norm of residual for LU %G\n",norm);CHKERRQ(ierr); } ierr = VecResetArray(b);CHKERRQ(ierr); ierr = VecResetArray(y);CHKERRQ(ierr); } ierr = MatDenseRestoreArray(RHS,&rhs_array);CHKERRQ(ierr); ierr = MatDenseRestoreArray(SOLU,&solu_array);CHKERRQ(ierr); ierr = MatDestroy(&F);CHKERRQ(ierr); /* out-place LU */ ierr = MatGetFactor(mat,MATSOLVERPETSC,MAT_FACTOR_LU,&F);CHKERRQ(ierr); ierr = MatLUFactorSymbolic(F,mat,0,0,0);CHKERRQ(ierr); ierr = MatLUFactorNumeric(F,mat,0);CHKERRQ(ierr); ierr = MatSolve(F,b,y);CHKERRQ(ierr); value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr); ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for LU %G\n",norm);CHKERRQ(ierr); } /* free space */ ierr = MatDestroy(&F);CHKERRQ(ierr); ierr = MatDestroy(&mat);CHKERRQ(ierr); ierr = MatDestroy(&RHS);CHKERRQ(ierr); ierr = MatDestroy(&SOLU);CHKERRQ(ierr); ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = VecDestroy(&y);CHKERRQ(ierr); ierr = VecDestroy(&ytmp);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
int main(int argc,char **args) { Vec x, b, u; /* approx solution, RHS, exact solution */ Mat A,B,C; /* linear system matrix */ KSP ksp; /* linear solver context */ PC pc; /* preconditioner context */ PetscReal norm; /* norm of solution error */ PetscErrorCode ierr; PetscInt i,n = 20,col[3],its; PetscMPIInt size; PetscScalar neg_one = -1.0,one = 1.0,value[3]; PetscBool nonzeroguess = PETSC_FALSE; PetscInitialize(&argc,&args,(char*)0,help); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); if (size != 1) SETERRQ(PETSC_COMM_WORLD,1,"This is a uniprocessor example only!"); ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetBool(NULL,"-nonzero_guess",&nonzeroguess,NULL);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Compute the matrix and right-hand-side vector that define the linear system, Ax = b. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create vectors. Note that we form 1 vector from scratch and then duplicate as needed. */ ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject) x, "Solution");CHKERRQ(ierr); ierr = VecSetSizes(x,PETSC_DECIDE,n);CHKERRQ(ierr); ierr = VecSetFromOptions(x);CHKERRQ(ierr); ierr = VecDuplicate(x,&b);CHKERRQ(ierr); ierr = VecDuplicate(x,&u);CHKERRQ(ierr); /* Create matrix. When using MatCreate(), the matrix format can be specified at runtime. Performance tuning note: For problems of substantial size, preallocation of matrix memory is crucial for attaining good performance. See the matrix chapter of the users manual for details. */ ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); ierr = MatSetUp(A);CHKERRQ(ierr); /* Assemble matrix */ value[0] = -1.0; value[1] = 2.0; value[2] = -1.0; for (i=1; i<n-1; i++) { col[0] = i-1; col[1] = i; col[2] = i+1; ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr); } i = n - 1; col[0] = n - 2; col[1] = n - 1; ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr); i = 0; col[0] = 0; col[1] = 1; value[0] = 2.0; value[1] = -1.0; ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* jroman: added matrices */ ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr); ierr = MatSetSizes(B,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr); ierr = MatSetFromOptions(B);CHKERRQ(ierr); ierr = MatSetUp(B);CHKERRQ(ierr); for (i=0; i<n; i++) { ierr = MatSetValue(B,i,i,value[1],INSERT_VALUES);CHKERRQ(ierr); if (n-i+n/3<n) { ierr = MatSetValue(B,n-i+n/3,i,value[0],INSERT_VALUES);CHKERRQ(ierr); ierr = MatSetValue(B,i,n-i+n/3,value[0],INSERT_VALUES);CHKERRQ(ierr); } } ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatDuplicate(A,MAT_COPY_VALUES,&C);CHKERRQ(ierr); ierr = MatAXPY(C,2.0,B,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr); /* Set exact solution; then compute right-hand-side vector. */ ierr = VecSet(u,one);CHKERRQ(ierr); ierr = MatMult(C,u,b);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create the linear solver and set various options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create linear solver context */ ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr); /* Set operators. Here the matrix that defines the linear system also serves as the preconditioning matrix. */ ierr = KSPSetOperators(ksp,C,C);CHKERRQ(ierr); /* Set linear solver defaults for this problem (optional). - By extracting the KSP and PC contexts from the KSP context, we can then directly call any KSP and PC routines to set various options. - The following four statements are optional; all of these parameters could alternatively be specified at runtime via KSPSetFromOptions(); */ ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr); ierr = PCSetType(pc,PCJACOBI);CHKERRQ(ierr); ierr = KSPSetTolerances(ksp,1.e-5,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);CHKERRQ(ierr); /* Set runtime options, e.g., -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol> These options will override those specified above as long as KSPSetFromOptions() is called _after_ any other customization routines. */ ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); if (nonzeroguess) { PetscScalar p = .5; ierr = VecSet(x,p);CHKERRQ(ierr); ierr = KSPSetInitialGuessNonzero(ksp,PETSC_TRUE);CHKERRQ(ierr); } /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Solve the linear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Solve linear system */ ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Check solution and clean up - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Check the error */ ierr = VecAXPY(x,neg_one,u);CHKERRQ(ierr); ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr); ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %g, Iterations %D\n",(double)norm,its);CHKERRQ(ierr); /* Free work space. All PETSc objects should be destroyed when they are no longer needed. */ ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = MatDestroy(&B);CHKERRQ(ierr); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = KSPDestroy(&ksp);CHKERRQ(ierr); /* Always call PetscFinalize() before exiting a program. This routine - finalizes the PETSc libraries as well as MPI - provides summary and diagnostic information if certain runtime options are chosen (e.g., -log_summary). */ ierr = PetscFinalize(); return 0; }
int main(int argc,char **argv) { PetscMPIInt rank,size; PetscInt nlocal = 6,nghost = 2,ifrom[2],i,rstart,rend; PetscErrorCode ierr; PetscBool flg,flg2; PetscScalar value,*array,*tarray=0; Vec lx,gx,gxs; ierr = PetscInitialize(&argc,&argv,(char*)0,help); if (ierr) return ierr; ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank); CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size); CHKERRQ(ierr); if (size != 2) SETERRQ(PETSC_COMM_SELF,1,"Must run example with two processors\n"); /* Construct a two dimensional graph connecting nlocal degrees of freedom per processor. From this we will generate the global indices of needed ghost values For simplicity we generate the entire graph on each processor: in real application the graph would stored in parallel, but this example is only to demonstrate the management of ghost padding with VecCreateGhost(). In this example we consider the vector as representing degrees of freedom in a one dimensional grid with periodic boundary conditions. ----Processor 1--------- ----Processor 2 -------- 0 1 2 3 4 5 6 7 8 9 10 11 |----| |-------------------------------------------------| */ if (!rank) { ifrom[0] = 11; ifrom[1] = 6; } else { ifrom[0] = 0; ifrom[1] = 5; } /* Create the vector with two slots for ghost points. Note that both the local vector (lx) and the global vector (gx) share the same array for storing vector values. */ ierr = PetscOptionsHasName(NULL,NULL,"-allocate",&flg); CHKERRQ(ierr); ierr = PetscOptionsHasName(NULL,NULL,"-vecmpisetghost",&flg2); CHKERRQ(ierr); if (flg) { ierr = PetscMalloc1(nlocal+nghost,&tarray); CHKERRQ(ierr); ierr = VecCreateGhostWithArray(PETSC_COMM_WORLD,nlocal,PETSC_DECIDE,nghost,ifrom,tarray,&gxs); CHKERRQ(ierr); } else if (flg2) { ierr = VecCreate(PETSC_COMM_WORLD,&gxs); CHKERRQ(ierr); ierr = VecSetType(gxs,VECMPI); CHKERRQ(ierr); ierr = VecSetSizes(gxs,nlocal,PETSC_DECIDE); CHKERRQ(ierr); ierr = VecMPISetGhost(gxs,nghost,ifrom); CHKERRQ(ierr); } else { ierr = VecCreateGhost(PETSC_COMM_WORLD,nlocal,PETSC_DECIDE,nghost,ifrom,&gxs); CHKERRQ(ierr); } /* Test VecDuplicate() */ ierr = VecDuplicate(gxs,&gx); CHKERRQ(ierr); ierr = VecDestroy(&gxs); CHKERRQ(ierr); /* Access the local representation */ ierr = VecGhostGetLocalForm(gx,&lx); CHKERRQ(ierr); /* Set the values from 0 to 12 into the "global" vector */ ierr = VecGetOwnershipRange(gx,&rstart,&rend); CHKERRQ(ierr); for (i=rstart; i<rend; i++) { value = (PetscScalar) i; ierr = VecSetValues(gx,1,&i,&value,INSERT_VALUES); CHKERRQ(ierr); } ierr = VecAssemblyBegin(gx); CHKERRQ(ierr); ierr = VecAssemblyEnd(gx); CHKERRQ(ierr); ierr = VecGhostUpdateBegin(gx,INSERT_VALUES,SCATTER_FORWARD); CHKERRQ(ierr); ierr = VecGhostUpdateEnd(gx,INSERT_VALUES,SCATTER_FORWARD); CHKERRQ(ierr); /* Print out each vector, including the ghost padding region. */ ierr = VecGetArray(lx,&array); CHKERRQ(ierr); for (i=0; i<nlocal+nghost; i++) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"%D %g\n",i,(double)PetscRealPart(array[i])); CHKERRQ(ierr); } ierr = VecRestoreArray(lx,&array); CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT); CHKERRQ(ierr); ierr = VecGhostRestoreLocalForm(gx,&lx); CHKERRQ(ierr); ierr = VecDestroy(&gx); CHKERRQ(ierr); if (flg) { ierr = PetscFree(tarray); CHKERRQ(ierr); } ierr = PetscFinalize(); return ierr; }
int main(int argc,char **args) { Mat A; Vec min,max,maxabs; PetscInt m,n; PetscInt imin[M],imax[M],imaxabs[M],indices[N],row; PetscScalar values[N]; PetscErrorCode ierr; MatType type; PetscMPIInt size; PetscBool doTest=PETSC_TRUE; ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr; ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,M,N);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); ierr = MatSetUp(A);CHKERRQ(ierr); row = 0; indices[0] = 0; indices[1] = 1; indices[2] = 2; indices[3] = 3; indices[4] = 4; indices[5] = 5; values[0] = -1.0; values[1] = 0.0; values[2] = 1.0; values[3] = 3.0; values[4] = 4.0; values[5] = -5.0; ierr = MatSetValues(A,1,&row,6,indices,values,INSERT_VALUES);CHKERRQ(ierr); row = 1; ierr = MatSetValues(A,1,&row,3,indices,values,INSERT_VALUES);CHKERRQ(ierr); row = 4; ierr = MatSetValues(A,1,&row,1,indices+4,values+4,INSERT_VALUES);CHKERRQ(ierr); row = 4; ierr = MatSetValues(A,1,&row,2,indices+4,values+4,INSERT_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = MatGetLocalSize(A, &m,&n);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_WORLD,&min);CHKERRQ(ierr); ierr = VecSetSizes(min,m,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(min);CHKERRQ(ierr); ierr = VecDuplicate(min,&max);CHKERRQ(ierr); ierr = VecDuplicate(min,&maxabs);CHKERRQ(ierr); /* Test MatGetRowMin, MatGetRowMax and MatGetRowMaxAbs */ if (size == 1) { ierr = MatGetRowMin(A,min,imin);CHKERRQ(ierr); ierr = MatGetRowMax(A,max,imax);CHKERRQ(ierr); ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Minimums\n");CHKERRQ(ierr); ierr = VecView(min,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscIntView(5,imin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximums\n");CHKERRQ(ierr); ierr = VecView(max,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscIntView(5,imax,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values\n");CHKERRQ(ierr); ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscIntView(5,imaxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } else { ierr = MatGetType(A,&type);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"\nMatrix type: %s\n",type);CHKERRQ(ierr); /* AIJ */ ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&doTest);CHKERRQ(ierr); if (doTest) { ierr = MatGetRowMaxAbs(A,maxabs,NULL);CHKERRQ(ierr); ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values:\n");CHKERRQ(ierr); ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* BAIJ */ ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIBAIJ,&doTest);CHKERRQ(ierr); if (doTest) { ierr = MatGetRowMaxAbs(A,maxabs,NULL);CHKERRQ(ierr); ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values:\n");CHKERRQ(ierr); ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } } if (size == 1) { ierr = MatConvert(A,MATDENSE,MAT_INPLACE_MATRIX,&A);CHKERRQ(ierr); ierr = MatGetRowMin(A,min,imin);CHKERRQ(ierr); ierr = MatGetRowMax(A,max,imax);CHKERRQ(ierr); ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr); ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Minimums\n");CHKERRQ(ierr); ierr = VecView(min,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscIntView(5,imin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximums\n");CHKERRQ(ierr); ierr = VecView(max,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscIntView(5,imax,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values\n");CHKERRQ(ierr); ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = PetscIntView(5,imaxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } ierr = VecDestroy(&min);CHKERRQ(ierr); ierr = VecDestroy(&max);CHKERRQ(ierr); ierr = VecDestroy(&maxabs);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
uses block index sets\n\n"; #include <petscvec.h> int main(int argc,char **argv) { PetscErrorCode ierr; PetscInt bs=1,n=5,N,i,low; PetscInt ix0[3] = {5,7,9},iy0[3] = {1,2,4},ix1[3] = {2,3,1},iy1[3] = {0,3,9}; PetscMPIInt size,rank; PetscScalar *array; Vec x,x1,y; IS isx,isy; VecScatter ctx; VecScatterType type; PetscBool flg; ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); if (size <2) SETERRQ(PETSC_COMM_SELF,1,"Must run more than one processor"); ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);CHKERRQ(ierr); n = bs*n; /* Create vector x over shared memory */ ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr); ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetType(x,VECNODE);CHKERRQ(ierr); ierr = VecSetFromOptions(x);CHKERRQ(ierr); ierr = VecGetOwnershipRange(x,&low,NULL);CHKERRQ(ierr); ierr = VecGetArray(x,&array);CHKERRQ(ierr); for (i=0; i<n; i++) { array[i] = (PetscScalar)(i + low); } ierr = VecRestoreArray(x,&array);CHKERRQ(ierr); /* ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */ /* Test some vector functions */ ierr = VecAssemblyBegin(x);CHKERRQ(ierr); ierr = VecAssemblyEnd(x);CHKERRQ(ierr); ierr = VecGetSize(x,&N);CHKERRQ(ierr); ierr = VecGetLocalSize(x,&n);CHKERRQ(ierr); ierr = VecDuplicate(x,&x1);CHKERRQ(ierr); ierr = VecCopy(x,x1);CHKERRQ(ierr); ierr = VecEqual(x,x1,&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PetscObjectComm((PetscObject)x),PETSC_ERR_ARG_WRONG,"x1 != x"); ierr = VecScale(x1,2.0);CHKERRQ(ierr); ierr = VecSet(x1,10.0);CHKERRQ(ierr); /* ierr = VecView(x1,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */ /* Create vector y over shared memory */ ierr = VecCreate(PETSC_COMM_WORLD,&y);CHKERRQ(ierr); ierr = VecSetSizes(y,n,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetType(y,VECNODE);CHKERRQ(ierr); ierr = VecSetFromOptions(y);CHKERRQ(ierr); ierr = VecGetArray(y,&array);CHKERRQ(ierr); for (i=0; i<n; i++) { array[i] = -(PetscScalar) (i + 100*rank); } ierr = VecRestoreArray(y,&array);CHKERRQ(ierr); ierr = VecAssemblyBegin(y);CHKERRQ(ierr); ierr = VecAssemblyEnd(y);CHKERRQ(ierr); /* ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */ /* Create two index sets */ if (!rank) { ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix0,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr); ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy0,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr); } else { ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix1,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr); ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy1,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr); } if (rank == 10) { ierr = PetscPrintf(PETSC_COMM_SELF,"\n[%d] isx:\n",rank);CHKERRQ(ierr); ierr = ISView(isx,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_SELF,"\n[%d] isy:\n",rank);CHKERRQ(ierr); ierr = ISView(isy,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); } /* Create Vector scatter */ ierr = VecScatterCreate(x,isx,y,isy,&ctx);CHKERRQ(ierr); ierr = VecScatterSetFromOptions(ctx);CHKERRQ(ierr); ierr = VecScatterGetType(ctx,&type);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"scatter type %s\n",type);CHKERRQ(ierr); /* Test forward vecscatter */ ierr = VecSet(y,0.0);CHKERRQ(ierr); ierr = VecScatterBegin(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"\nSCATTER_FORWARD y:\n");CHKERRQ(ierr); ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* Test reverse vecscatter */ ierr = VecSet(x,0.0);CHKERRQ(ierr); ierr = VecSet(y,0.0);CHKERRQ(ierr); ierr = VecGetOwnershipRange(y,&low,NULL);CHKERRQ(ierr); ierr = VecGetArray(y,&array);CHKERRQ(ierr); for (i=0; i<n; i++) { array[i] = (PetscScalar)(i+ low); } ierr = VecRestoreArray(y,&array);CHKERRQ(ierr); ierr = VecScatterBegin(ctx,y,x,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(ctx,y,x,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"\nSCATTER_REVERSE x:\n");CHKERRQ(ierr); ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* Free objects */ ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr); ierr = ISDestroy(&isx);CHKERRQ(ierr); ierr = ISDestroy(&isy);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&x1);CHKERRQ(ierr); ierr = VecDestroy(&y);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
/*@ MatCreateSubMatrix - Creates a composite matrix that acts as a submatrix Collective on Mat Input Parameters: + A - matrix that we will extract a submatrix of . isrow - rows to be present in the submatrix - iscol - columns to be present in the submatrix Output Parameters: . newmat - new matrix Level: developer Notes: Most will use MatGetSubMatrix which provides a more efficient representation if it is available. .seealso: MatGetSubMatrix(), MatSubMatrixUpdate() @*/ PetscErrorCode MatCreateSubMatrix(Mat A,IS isrow,IS iscol,Mat *newmat) { Vec left,right; PetscInt m,n; Mat N; Mat_SubMatrix *Na; PetscErrorCode ierr; PetscFunctionBegin; PetscValidHeaderSpecific(A,MAT_CLASSID,1); PetscValidHeaderSpecific(isrow,IS_CLASSID,2); PetscValidHeaderSpecific(iscol,IS_CLASSID,3); PetscValidPointer(newmat,4); *newmat = 0; ierr = MatCreate(PetscObjectComm((PetscObject)A),&N);CHKERRQ(ierr); ierr = ISGetLocalSize(isrow,&m);CHKERRQ(ierr); ierr = ISGetLocalSize(iscol,&n);CHKERRQ(ierr); ierr = MatSetSizes(N,m,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr); ierr = PetscObjectChangeTypeName((PetscObject)N,MATSUBMATRIX);CHKERRQ(ierr); ierr = PetscNewLog(N,&Na);CHKERRQ(ierr); N->data = (void*)Na; ierr = PetscObjectReference((PetscObject)A);CHKERRQ(ierr); ierr = PetscObjectReference((PetscObject)isrow);CHKERRQ(ierr); ierr = PetscObjectReference((PetscObject)iscol);CHKERRQ(ierr); Na->A = A; Na->isrow = isrow; Na->iscol = iscol; Na->scale = 1.0; N->ops->destroy = MatDestroy_SubMatrix; N->ops->mult = MatMult_SubMatrix; N->ops->multadd = MatMultAdd_SubMatrix; N->ops->multtranspose = MatMultTranspose_SubMatrix; N->ops->multtransposeadd = MatMultTransposeAdd_SubMatrix; N->ops->scale = MatScale_SubMatrix; N->ops->diagonalscale = MatDiagonalScale_SubMatrix; ierr = MatSetBlockSizesFromMats(N,A,A);CHKERRQ(ierr); ierr = PetscLayoutSetUp(N->rmap);CHKERRQ(ierr); ierr = PetscLayoutSetUp(N->cmap);CHKERRQ(ierr); ierr = MatCreateVecs(A,&Na->rwork,&Na->lwork);CHKERRQ(ierr); ierr = VecCreate(PetscObjectComm((PetscObject)isrow),&left);CHKERRQ(ierr); ierr = VecCreate(PetscObjectComm((PetscObject)iscol),&right);CHKERRQ(ierr); ierr = VecSetSizes(left,m,PETSC_DETERMINE);CHKERRQ(ierr); ierr = VecSetSizes(right,n,PETSC_DETERMINE);CHKERRQ(ierr); ierr = VecSetUp(left);CHKERRQ(ierr); ierr = VecSetUp(right);CHKERRQ(ierr); ierr = VecScatterCreate(Na->lwork,isrow,left,NULL,&Na->lrestrict);CHKERRQ(ierr); ierr = VecScatterCreate(right,NULL,Na->rwork,iscol,&Na->rprolong);CHKERRQ(ierr); ierr = VecDestroy(&left);CHKERRQ(ierr); ierr = VecDestroy(&right);CHKERRQ(ierr); N->assembled = PETSC_TRUE; ierr = MatSetUp(N);CHKERRQ(ierr); *newmat = N; PetscFunctionReturn(0); }
int main(int argc,char **argv) { Mat A,F,B,X,C,Aher,G; Vec b,x,c,d,e; PetscErrorCode ierr; PetscInt m = 5,n,p,i,j,nrows,ncols; PetscScalar *v,*barray,rval; PetscReal norm,tol=1.e-12; PetscMPIInt size,rank; PetscRandom rand; const PetscInt *rows,*cols; IS isrows,iscols; PetscBool mats_view=PETSC_FALSE; MatFactorInfo finfo; PetscInitialize(&argc,&argv,(char*) 0,help); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr); /* Get local dimensions of matrices */ ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr); n = m; ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr); p = m/2; ierr = PetscOptionsGetInt(NULL,"-p",&p,NULL);CHKERRQ(ierr); ierr = PetscOptionsHasName(NULL,"-mats_view",&mats_view);CHKERRQ(ierr); /* Create matrix A */ ierr = PetscPrintf(PETSC_COMM_WORLD," Create Elemental matrix A\n");CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr); ierr = MatSetType(A,MATELEMENTAL);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); ierr = MatSetUp(A);CHKERRQ(ierr); /* Set local matrix entries */ ierr = MatGetOwnershipIS(A,&isrows,&iscols);CHKERRQ(ierr); ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr); ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr); ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr); ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr); ierr = PetscMalloc1(nrows*ncols,&v);CHKERRQ(ierr); for (i=0; i<nrows; i++) { for (j=0; j<ncols; j++) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); v[i*ncols+j] = rval; } } ierr = MatSetValues(A,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr); ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr); ierr = ISDestroy(&isrows);CHKERRQ(ierr); ierr = ISDestroy(&iscols);CHKERRQ(ierr); ierr = PetscFree(v);CHKERRQ(ierr); if (mats_view) { ierr = PetscPrintf(PETSC_COMM_WORLD, "A: nrows %d, m %d; ncols %d, n %d\n",nrows,m,ncols,n);CHKERRQ(ierr); ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Create rhs matrix B */ ierr = PetscPrintf(PETSC_COMM_WORLD," Create rhs matrix B\n");CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr); ierr = MatSetSizes(B,m,p,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr); ierr = MatSetType(B,MATELEMENTAL);CHKERRQ(ierr); ierr = MatSetFromOptions(B);CHKERRQ(ierr); ierr = MatSetUp(B);CHKERRQ(ierr); ierr = MatGetOwnershipIS(B,&isrows,&iscols);CHKERRQ(ierr); ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr); ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr); ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr); ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr); ierr = PetscMalloc1(nrows*ncols,&v);CHKERRQ(ierr); for (i=0; i<nrows; i++) { for (j=0; j<ncols; j++) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); v[i*ncols+j] = rval; } } ierr = MatSetValues(B,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr); ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr); ierr = ISDestroy(&isrows);CHKERRQ(ierr); ierr = ISDestroy(&iscols);CHKERRQ(ierr); ierr = PetscFree(v);CHKERRQ(ierr); if (mats_view) { ierr = PetscPrintf(PETSC_COMM_WORLD, "B: nrows %d, m %d; ncols %d, p %d\n",nrows,m,ncols,p);CHKERRQ(ierr); ierr = MatView(B,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Create rhs vector b and solution x (same size as b) */ ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr); ierr = VecSetSizes(b,m,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(b);CHKERRQ(ierr); ierr = VecGetArray(b,&barray);CHKERRQ(ierr); for (j=0; j<m; j++) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); barray[j] = rval; } ierr = VecRestoreArray(b,&barray);CHKERRQ(ierr); ierr = VecAssemblyBegin(b);CHKERRQ(ierr); ierr = VecAssemblyEnd(b);CHKERRQ(ierr); if (mats_view) { ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d] b: m %d\n",rank,m);CHKERRQ(ierr); ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr); ierr = VecView(b,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } ierr = VecDuplicate(b,&x);CHKERRQ(ierr); /* Create matrix X - same size as B */ ierr = PetscPrintf(PETSC_COMM_WORLD," Create solution matrix X\n");CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&X);CHKERRQ(ierr); ierr = MatSetSizes(X,m,p,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr); ierr = MatSetType(X,MATELEMENTAL);CHKERRQ(ierr); ierr = MatSetFromOptions(X);CHKERRQ(ierr); ierr = MatSetUp(X);CHKERRQ(ierr); ierr = MatAssemblyBegin(X,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(X,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* Cholesky factorization */ /*------------------------*/ ierr = PetscPrintf(PETSC_COMM_WORLD," Create Elemental matrix Aher\n");CHKERRQ(ierr); ierr = MatHermitianTranspose(A,MAT_INITIAL_MATRIX,&Aher);CHKERRQ(ierr); ierr = MatAXPY(Aher,1.0,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr); /* Aher = A + A^T */ if (!rank) { /* add 100.0 to diagonals of Aher to make it spd */ PetscInt M,N; ierr = MatGetSize(Aher,&M,&N);CHKERRQ(ierr); for (i=0; i<M; i++) { rval = 100.0; ierr = MatSetValues(Aher,1,&i,1,&i,&rval,ADD_VALUES);CHKERRQ(ierr); } } ierr = MatAssemblyBegin(Aher,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(Aher,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); if (mats_view) { ierr = PetscPrintf(PETSC_COMM_WORLD, "Aher:\n");CHKERRQ(ierr); ierr = MatView(Aher,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Cholesky factorization */ /*------------------------*/ ierr = PetscPrintf(PETSC_COMM_WORLD," Test Cholesky Solver \n");CHKERRQ(ierr); /* In-place Cholesky */ /* Create matrix factor G, then copy Aher to G */ ierr = MatCreate(PETSC_COMM_WORLD,&G);CHKERRQ(ierr); ierr = MatSetSizes(G,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr); ierr = MatSetType(G,MATELEMENTAL);CHKERRQ(ierr); ierr = MatSetFromOptions(G);CHKERRQ(ierr); ierr = MatSetUp(G);CHKERRQ(ierr); ierr = MatAssemblyBegin(G,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(G,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatCopy(Aher,G,SAME_NONZERO_PATTERN);CHKERRQ(ierr); /* Only G = U^T * U is implemented for now */ ierr = MatCholeskyFactor(G,0,0);CHKERRQ(ierr); if (mats_view) { ierr = PetscPrintf(PETSC_COMM_WORLD, "Cholesky Factor G:\n");CHKERRQ(ierr); ierr = MatView(G,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Solve U^T * U x = b and U^T * U X = B */ ierr = MatSolve(G,b,x);CHKERRQ(ierr); ierr = MatMatSolve(G,B,X);CHKERRQ(ierr); ierr = MatDestroy(&G);CHKERRQ(ierr); /* Out-place Cholesky */ ierr = MatGetFactor(Aher,MATSOLVERELEMENTAL,MAT_FACTOR_CHOLESKY,&G);CHKERRQ(ierr); ierr = MatCholeskyFactorSymbolic(G,Aher,0,&finfo);CHKERRQ(ierr); ierr = MatCholeskyFactorNumeric(G,Aher,&finfo);CHKERRQ(ierr); if (mats_view) { ierr = MatView(G,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } ierr = MatSolve(G,b,x);CHKERRQ(ierr); ierr = MatMatSolve(G,B,X);CHKERRQ(ierr); ierr = MatDestroy(&G);CHKERRQ(ierr); /* Check norm(Aher*x - b) */ ierr = VecCreate(PETSC_COMM_WORLD,&c);CHKERRQ(ierr); ierr = VecSetSizes(c,m,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(c);CHKERRQ(ierr); ierr = MatMult(Aher,x,c);CHKERRQ(ierr); ierr = VecAXPY(c,-1.0,b);CHKERRQ(ierr); ierr = VecNorm(c,NORM_1,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |Aher*x - b| for Cholesky %G\n",norm);CHKERRQ(ierr); } /* Check norm(Aher*X - B) */ ierr = MatMatMult(Aher,X,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&C);CHKERRQ(ierr); ierr = MatAXPY(C,-1.0,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr); ierr = MatNorm(C,NORM_1,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |Aher*X - B| for Cholesky %G\n",norm);CHKERRQ(ierr); } /* LU factorization */ /*------------------*/ ierr = PetscPrintf(PETSC_COMM_WORLD," Test LU Solver \n");CHKERRQ(ierr); /* In-place LU */ /* Create matrix factor F, then copy A to F */ ierr = MatCreate(PETSC_COMM_WORLD,&F);CHKERRQ(ierr); ierr = MatSetSizes(F,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr); ierr = MatSetType(F,MATELEMENTAL);CHKERRQ(ierr); ierr = MatSetFromOptions(F);CHKERRQ(ierr); ierr = MatSetUp(F);CHKERRQ(ierr); ierr = MatAssemblyBegin(F,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(F,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatCopy(A,F,SAME_NONZERO_PATTERN);CHKERRQ(ierr); /* Create vector d to test MatSolveAdd() */ ierr = VecDuplicate(x,&d);CHKERRQ(ierr); ierr = VecCopy(x,d);CHKERRQ(ierr); /* PF=LU or F=LU factorization - perms is ignored by Elemental; set finfo.dtcol !0 or 0 to enable/disable partial pivoting */ finfo.dtcol = 0.1; ierr = MatLUFactor(F,0,0,&finfo);CHKERRQ(ierr); /* Solve LUX = PB or LUX = B */ ierr = MatSolveAdd(F,b,d,x);CHKERRQ(ierr); ierr = MatMatSolve(F,B,X);CHKERRQ(ierr); ierr = MatDestroy(&F);CHKERRQ(ierr); /* Check norm(A*X - B) */ ierr = VecCreate(PETSC_COMM_WORLD,&e);CHKERRQ(ierr); ierr = VecSetSizes(e,m,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(e);CHKERRQ(ierr); ierr = MatMult(A,x,c);CHKERRQ(ierr); ierr = MatMult(A,d,e);CHKERRQ(ierr); ierr = VecAXPY(c,-1.0,e);CHKERRQ(ierr); ierr = VecAXPY(c,-1.0,b);CHKERRQ(ierr); ierr = VecNorm(c,NORM_1,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |A*x - b| for LU %G\n",norm);CHKERRQ(ierr); } ierr = MatMatMult(A,X,MAT_REUSE_MATRIX,PETSC_DEFAULT,&C);CHKERRQ(ierr); ierr = MatAXPY(C,-1.0,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr); ierr = MatNorm(C,NORM_1,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |A*X - B| for LU %G\n",norm);CHKERRQ(ierr); } /* Out-place LU */ ierr = MatGetFactor(A,MATSOLVERELEMENTAL,MAT_FACTOR_LU,&F);CHKERRQ(ierr); ierr = MatLUFactorSymbolic(F,A,0,0,&finfo);CHKERRQ(ierr); ierr = MatLUFactorNumeric(F,A,&finfo);CHKERRQ(ierr); if (mats_view) { ierr = MatView(F,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } ierr = MatSolve(F,b,x);CHKERRQ(ierr); ierr = MatMatSolve(F,B,X);CHKERRQ(ierr); ierr = MatDestroy(&F);CHKERRQ(ierr); /* Free space */ ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = MatDestroy(&Aher);CHKERRQ(ierr); ierr = MatDestroy(&B);CHKERRQ(ierr); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = MatDestroy(&X);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = VecDestroy(&c);CHKERRQ(ierr); ierr = VecDestroy(&d);CHKERRQ(ierr); ierr = VecDestroy(&e);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
int main(int argc,char **args) { Mat C; PetscInt i,j,m = 5,n = 2,Ii,J; PetscErrorCode ierr; PetscMPIInt rank,size; PetscScalar v; Vec x,y; PetscInitialize(&argc,&args,(char *)0,help); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); n = 2*size; /* Create the matrix for the five point stencil, YET AGAIN */ ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr); ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr); ierr = MatSetFromOptions(C);CHKERRQ(ierr); for (i=0; i<m; i++) { for (j=2*rank; j<2*rank+2; j++) { v = -1.0; Ii = j + n*i; if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); for (i=0; i<m; i++) { for (j=2*rank; j<2*rank+2; j++) { v = 1.0; Ii = j + n*i; if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = -4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } /* Introduce new nonzero that requires new construction for matrix-vector product */ if (rank) { Ii = rank-1; J = m*n-1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); } ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* Form a couple of vectors to test matrix-vector product */ ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr); ierr = VecSetSizes(x,PETSC_DECIDE,m*n);CHKERRQ(ierr); ierr = VecSetFromOptions(x);CHKERRQ(ierr); ierr = VecDuplicate(x,&y);CHKERRQ(ierr); v = 1.0; ierr = VecSet(x,v);CHKERRQ(ierr); ierr = MatMult(C,x,y);CHKERRQ(ierr); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&y);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
PetscErrorCode PipesView(Vec X,DM networkdm,Wash wash) { PetscErrorCode ierr; Pipe pipe; DMNetworkComponentGenericDataType *nwarr; PetscInt pipeOffset,key,Start,End; PetscMPIInt rank; PetscInt nx,nnodes,nidx,*idx1,*idx2,*idx1_h,*idx2_h,idx_start,i,k,k1,xstart,j1; Vec Xq,Xh,localX; IS is1_q,is2_q,is1_h,is2_h; VecScatter ctx_q,ctx_h; PetscFunctionBegin; ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); /* get num of local and global total nnodes */ nidx = wash->nnodes_loc; ierr = MPIU_Allreduce(&nidx,&nx,1,MPIU_INT,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_WORLD,&Xq);CHKERRQ(ierr); if (rank == 0) { /* all entries of Xq are in proc[0] */ ierr = VecSetSizes(Xq,nx,PETSC_DECIDE);CHKERRQ(ierr); } else { ierr = VecSetSizes(Xq,0,PETSC_DECIDE);CHKERRQ(ierr); } ierr = VecSetFromOptions(Xq);CHKERRQ(ierr); ierr = VecSet(Xq,0.0);CHKERRQ(ierr); ierr = VecDuplicate(Xq,&Xh);CHKERRQ(ierr); ierr = DMGetLocalVector(networkdm,&localX);CHKERRQ(ierr); /* set idx1 and idx2 */ ierr = PetscCalloc4(nidx,&idx1,nidx,&idx2,nidx,&idx1_h,nidx,&idx2_h);CHKERRQ(ierr); ierr = DMNetworkGetComponentDataArray(networkdm,&nwarr);CHKERRQ(ierr); ierr = DMNetworkGetEdgeRange(networkdm,&Start, &End);CHKERRQ(ierr); ierr = VecGetOwnershipRange(X,&xstart,NULL);CHKERRQ(ierr); k1 = 0; j1 = 0; for (i = Start; i < End; i++) { ierr = DMNetworkGetComponentTypeOffset(networkdm,i,0,&key,&pipeOffset);CHKERRQ(ierr); pipe = (Pipe)(nwarr+pipeOffset); nnodes = pipe->nnodes; idx_start = pipe->id*nnodes; for (k=0; k<nnodes; k++) { idx1[k1] = xstart + j1*2*nnodes + 2*k; idx2[k1] = idx_start + k; idx1_h[k1] = xstart + j1*2*nnodes + 2*k + 1; idx2_h[k1] = idx_start + k; k1++; } j1++; } ierr = ISCreateGeneral(PETSC_COMM_SELF,nidx,idx1,PETSC_COPY_VALUES,&is1_q);CHKERRQ(ierr); ierr = ISCreateGeneral(PETSC_COMM_SELF,nidx,idx2,PETSC_COPY_VALUES,&is2_q);CHKERRQ(ierr); ierr = VecScatterCreate(X,is1_q,Xq,is2_q,&ctx_q);CHKERRQ(ierr); ierr = VecScatterBegin(ctx_q,X,Xq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(ctx_q,X,Xq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = ISCreateGeneral(PETSC_COMM_SELF,nidx,idx1_h,PETSC_COPY_VALUES,&is1_h);CHKERRQ(ierr); ierr = ISCreateGeneral(PETSC_COMM_SELF,nidx,idx2_h,PETSC_COPY_VALUES,&is2_h);CHKERRQ(ierr); ierr = VecScatterCreate(X,is1_h,Xh,is2_h,&ctx_h);CHKERRQ(ierr); ierr = VecScatterBegin(ctx_h,X,Xh,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(ctx_h,X,Xh,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); if (!rank) printf("Xq: \n"); ierr = VecView(Xq,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); if (!rank) printf("Xh: \n"); ierr = VecView(Xh,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = VecScatterDestroy(&ctx_q);CHKERRQ(ierr); ierr = PetscFree4(idx1,idx2,idx1_h,idx2_h);CHKERRQ(ierr); ierr = ISDestroy(&is1_q);CHKERRQ(ierr); ierr = ISDestroy(&is2_q);CHKERRQ(ierr); ierr = VecScatterDestroy(&ctx_h);CHKERRQ(ierr); ierr = ISDestroy(&is1_h);CHKERRQ(ierr); ierr = ISDestroy(&is2_h);CHKERRQ(ierr); ierr = VecDestroy(&Xq);CHKERRQ(ierr); ierr = VecDestroy(&Xh);CHKERRQ(ierr); ierr = DMRestoreLocalVector(networkdm,&localX);CHKERRQ(ierr); PetscFunctionReturn(0); }
int PetscSOE::setSizeParallel(int newSize, int N, int d_nz, int *d_nnz, int o_nz, int *o_nnz) { int result = 0; int oldSize = size; size = newSize; isFactored = 0; // delete the old if (B != 0) delete [] B; if (X != 0) delete [] X; // invoke the petsc destructors if (A != 0) MatDestroy(A); if (b != 0) VecDestroy(b); if (x != 0) VecDestroy(x); // create the new B = new double[size]; X = new double[size]; if (B == 0 || X == 0) { opserr << "WARNING PetscSOE::PetscSOE :"; opserr << " ran out of memory for vectors (size) ("; opserr << size << ") \n"; size = 0; result = -1; } // zero the vectors for (int j=0; j<size; j++) { B[j] = 0; X[j] = 0; } if (vectX != 0) delete vectX; if (vectB != 0) delete vectB; vectX = new Vector(X,size); vectB = new Vector(B,size); // invoke the petsc constructors int ierr = OptionsGetInt(PETSC_NULL,"-n",&size,&flg); CHKERRA(ierr); ierr = MatCreate(PETSC_COMM_WORLD,size,size,&A); CHKERRA(ierr); ierr = VecCreate(PETSC_COMM_WORLD,PETSC_DECIDE,size,&x); CHKERRA(ierr); ierr = VecDuplicate(x,&b); CHKERRA(ierr); // invoke setSize() on the Solver int solverOK = theSolver->setSize(); if (solverOK < 0) { opserr << "WARNING:PetscSOE::setSize :"; opserr << " solver failed setSize()\n"; return solverOK; } return result; }
int main(int argc,char **argv) { TS ts; /* ODE integrator */ Vec U; /* solution will be stored here */ PetscErrorCode ierr; PetscMPIInt size; PetscInt n = 2; PetscScalar *u; AppCtx app; PetscInt direction[2]; PetscBool terminate[2]; PetscBool rhs_form=PETSC_FALSE,hist=PETSC_TRUE; TSAdapt adapt; /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Initialize program - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); if (size > 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Only for sequential runs"); app.nbounces = 0; app.maxbounces = 10; ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"ex40 options","");CHKERRQ(ierr); ierr = PetscOptionsInt("-maxbounces","","",app.maxbounces,&app.maxbounces,NULL);CHKERRQ(ierr); ierr = PetscOptionsBool("-test_adapthistory","","",hist,&hist,NULL);CHKERRQ(ierr); ierr = PetscOptionsEnd();CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create timestepping solver context - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr); ierr = TSSetType(ts,TSROSW);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Set ODE routines - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr); /* Users are advised against the following branching and code duplication. For problems without a mass matrix like the one at hand, the RHSFunction (and companion RHSJacobian) interface is enough to support both explicit and implicit timesteppers. This tutorial example also deals with the IFunction/IJacobian interface for demonstration and testing purposes. */ ierr = PetscOptionsGetBool(NULL,NULL,"-rhs-form",&rhs_form,NULL);CHKERRQ(ierr); if (rhs_form) { ierr = TSSetRHSFunction(ts,NULL,RHSFunction,NULL);CHKERRQ(ierr); ierr = TSSetRHSJacobian(ts,NULL,NULL,RHSJacobian,NULL);CHKERRQ(ierr); } else { Mat A; /* Jacobian matrix */ ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr); ierr = MatSetType(A,MATDENSE);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); ierr = MatSetUp(A);CHKERRQ(ierr); ierr = TSSetIFunction(ts,NULL,IFunction,NULL);CHKERRQ(ierr); ierr = TSSetIJacobian(ts,A,A,IJacobian,NULL);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); } /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Set initial conditions - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = VecCreate(PETSC_COMM_WORLD,&U);CHKERRQ(ierr); ierr = VecSetSizes(U,n,PETSC_DETERMINE);CHKERRQ(ierr); ierr = VecSetUp(U);CHKERRQ(ierr); ierr = VecGetArray(U,&u);CHKERRQ(ierr); u[0] = 0.0; u[1] = 20.0; ierr = VecRestoreArray(U,&u);CHKERRQ(ierr); ierr = TSSetSolution(ts,U);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Set solver options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = TSSetSaveTrajectory(ts);CHKERRQ(ierr); ierr = TSSetMaxTime(ts,30.0);CHKERRQ(ierr); ierr = TSSetExactFinalTime(ts,TS_EXACTFINALTIME_STEPOVER);CHKERRQ(ierr); ierr = TSSetTimeStep(ts,0.1);CHKERRQ(ierr); /* The adapative time step controller could take very large timesteps resulting in the same event occuring multiple times in the same interval. A maximum step size limit is enforced here to avoid this issue. */ ierr = TSGetAdapt(ts,&adapt);CHKERRQ(ierr); ierr = TSAdaptSetStepLimits(adapt,0.0,0.5);CHKERRQ(ierr); /* Set directions and terminate flags for the two events */ direction[0] = -1; direction[1] = -1; terminate[0] = PETSC_FALSE; terminate[1] = PETSC_TRUE; ierr = TSSetEventHandler(ts,2,direction,terminate,EventFunction,PostEventFunction,(void*)&app);CHKERRQ(ierr); ierr = TSSetFromOptions(ts);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Run timestepping solver - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = TSSolve(ts,U);CHKERRQ(ierr); if (hist) { /* replay following history */ TSTrajectory tj; PetscReal tf,t0,dt; app.nbounces = 0; ierr = TSGetTime(ts,&tf);CHKERRQ(ierr); ierr = TSSetMaxTime(ts,tf);CHKERRQ(ierr); ierr = TSSetStepNumber(ts,0);CHKERRQ(ierr); ierr = TSRestartStep(ts);CHKERRQ(ierr); ierr = TSSetExactFinalTime(ts,TS_EXACTFINALTIME_MATCHSTEP);CHKERRQ(ierr); ierr = TSSetFromOptions(ts);CHKERRQ(ierr); ierr = TSGetAdapt(ts,&adapt);CHKERRQ(ierr); ierr = TSAdaptSetType(adapt,TSADAPTHISTORY);CHKERRQ(ierr); ierr = TSGetTrajectory(ts,&tj);CHKERRQ(ierr); ierr = TSAdaptHistorySetTrajectory(adapt,tj,PETSC_FALSE);CHKERRQ(ierr); ierr = TSAdaptHistoryGetStep(adapt,0,&t0,&dt);CHKERRQ(ierr); /* this example fails with single (or smaller) precision */ #if defined(PETSC_USE_REAL_SINGLE) || defined(PETSC_USE_REAL__FP16) ierr = TSAdaptSetType(adapt,TSADAPTBASIC);CHKERRQ(ierr); ierr = TSAdaptSetStepLimits(adapt,0.0,0.5);CHKERRQ(ierr); ierr = TSSetFromOptions(ts);CHKERRQ(ierr); #endif ierr = TSSetTime(ts,t0);CHKERRQ(ierr); ierr = TSSetTimeStep(ts,dt);CHKERRQ(ierr); ierr = TSResetTrajectory(ts);CHKERRQ(ierr); ierr = VecGetArray(U,&u);CHKERRQ(ierr); u[0] = 0.0; u[1] = 20.0; ierr = VecRestoreArray(U,&u);CHKERRQ(ierr); ierr = TSSolve(ts,U);CHKERRQ(ierr); } /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Free work space. All PETSc objects should be destroyed when they are no longer needed. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = VecDestroy(&U);CHKERRQ(ierr); ierr = TSDestroy(&ts);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
int main(int argc,char **argv) { SNES snes; /* SNES context */ Vec x,r,F; /* vectors */ Mat J; /* Jacobian */ PetscErrorCode ierr; PetscInt it,n = 11,i; PetscReal h,xp = 0.0; PetscScalar v; const PetscScalar a = X0DOT; const PetscScalar b = X1; const PetscScalar k = KPOW; PetscScalar v2; PetscScalar *xx; PetscInitialize(&argc,&argv,(char*)0,help); ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetBool(NULL,NULL,"-second_order",&second_order,NULL);CHKERRQ(ierr); h = 1.0/(n-1); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create nonlinear solver context - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create vector data structures; set function evaluation routine - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = VecCreate(PETSC_COMM_SELF,&x);CHKERRQ(ierr); ierr = VecSetSizes(x,PETSC_DECIDE,n);CHKERRQ(ierr); ierr = VecSetFromOptions(x);CHKERRQ(ierr); ierr = VecDuplicate(x,&r);CHKERRQ(ierr); ierr = VecDuplicate(x,&F);CHKERRQ(ierr); ierr = SNESSetFunction(snes,r,FormFunction,(void*)F);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create matrix data structures; set Jacobian evaluation routine - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,n,n,3,NULL,&J);CHKERRQ(ierr); /* Note that in this case we create separate matrices for the Jacobian and preconditioner matrix. Both of these are computed in the routine FormJacobian() */ /* ierr = SNESSetJacobian(snes,NULL,JPrec,FormJacobian,0);CHKERRQ(ierr); */ ierr = SNESSetJacobian(snes,J,J,FormJacobian,0);CHKERRQ(ierr); /* ierr = SNESSetJacobian(snes,J,JPrec,FormJacobian,0);CHKERRQ(ierr); */ /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Customize nonlinear solver; set runtime options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = SNESSetFromOptions(snes);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Initialize application: Store right-hand-side of PDE and exact solution - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* set right hand side and initial guess to be exact solution of continuim problem */ #define SQR(x) ((x)*(x)) xp = 0.0; for (i=0; i<n; i++) { v = k*(k-1.)*(b-a)*PetscPowScalar(xp,k-2.) + SQR(a*xp) + SQR(b-a)*PetscPowScalar(xp,2.*k) + 2.*a*(b-a)*PetscPowScalar(xp,k+1.); ierr = VecSetValues(F,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr); v2 = a*xp + (b-a)*PetscPowScalar(xp,k); ierr = VecSetValues(x,1,&i,&v2,INSERT_VALUES);CHKERRQ(ierr); xp += h; } /* perturb initial guess */ ierr = VecGetArray(x,&xx); for (i=0; i<n; i++) { v2 = xx[i]*sperturb; ierr = VecSetValues(x,1,&i,&v2,INSERT_VALUES);CHKERRQ(ierr); } ierr = VecRestoreArray(x,&xx);CHKERRQ(ierr); ierr = SNESSolve(snes,NULL,x);CHKERRQ(ierr); ierr = SNESGetIterationNumber(snes,&it);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_SELF,"SNES iterations = %D\n\n",it);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Free work space. All PETSc objects should be destroyed when they are no longer needed. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&r);CHKERRQ(ierr); ierr = VecDestroy(&F);CHKERRQ(ierr); ierr = MatDestroy(&J);CHKERRQ(ierr); ierr = SNESDestroy(&snes);CHKERRQ(ierr); ierr = PetscFinalize();CHKERRQ(ierr); return 0; }
/*@ DMPlexCreateGmsh - Create a DMPlex mesh from a Gmsh file viewer Collective on comm Input Parameters: + comm - The MPI communicator . viewer - The Viewer associated with a Gmsh file - interpolate - Create faces and edges in the mesh Output Parameter: . dm - The DM object representing the mesh Note: http://www.geuz.org/gmsh/doc/texinfo/#MSH-ASCII-file-format and http://www.geuz.org/gmsh/doc/texinfo/#MSH-binary-file-format Level: beginner .keywords: mesh,Gmsh .seealso: DMPLEX, DMCreate() @*/ PetscErrorCode DMPlexCreateGmsh(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm) { PetscViewerType vtype; GmshElement *gmsh_elem; PetscSection coordSection; Vec coordinates; PetscScalar *coords, *coordsIn = NULL; PetscInt dim = 0, coordSize, c, v, d, r, cell; int i, numVertices = 0, numCells = 0, trueNumCells = 0, numRegions = 0, snum; PetscMPIInt num_proc, rank; char line[PETSC_MAX_PATH_LEN]; PetscBool match, binary, bswap = PETSC_FALSE; PetscErrorCode ierr; PetscFunctionBegin; ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr); ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr); ierr = DMCreate(comm, dm);CHKERRQ(ierr); ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr); ierr = PetscLogEventBegin(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr); ierr = PetscViewerGetType(viewer, &vtype);CHKERRQ(ierr); ierr = PetscStrcmp(vtype, PETSCVIEWERBINARY, &binary);CHKERRQ(ierr); if (!rank || binary) { PetscBool match; int fileType, dataSize; float version; /* Read header */ ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr); ierr = PetscStrncmp(line, "$MeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr); if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); ierr = PetscViewerRead(viewer, line, 3, NULL, PETSC_STRING);CHKERRQ(ierr); snum = sscanf(line, "%f %d %d", &version, &fileType, &dataSize); if (snum != 3) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Gmsh file header: %s", line); if (version < 2.0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Gmsh file must be at least version 2.0"); if (dataSize != sizeof(double)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Data size %d is not valid for a Gmsh file", dataSize); if (binary) { int checkInt; ierr = PetscViewerRead(viewer, &checkInt, 1, NULL, PETSC_ENUM);CHKERRQ(ierr); if (checkInt != 1) { ierr = PetscByteSwap(&checkInt, PETSC_ENUM, 1);CHKERRQ(ierr); if (checkInt == 1) bswap = PETSC_TRUE; else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh binary file", fileType); } } else if (fileType) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh ASCII file", fileType); ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr); ierr = PetscStrncmp(line, "$EndMeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr); if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); /* OPTIONAL Read physical names */ ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr); ierr = PetscStrncmp(line, "$PhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr); if (match) { ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr); snum = sscanf(line, "%d", &numRegions); if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); for (r = 0; r < numRegions; ++r) { PetscInt rdim, tag; ierr = PetscViewerRead(viewer, &rdim, 1, NULL, PETSC_ENUM);CHKERRQ(ierr); ierr = PetscViewerRead(viewer, &tag, 1, NULL, PETSC_ENUM);CHKERRQ(ierr); ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr); } ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr); ierr = PetscStrncmp(line, "$EndPhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr); if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); /* Initial read for vertex section */ ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr); } /* Read vertices */ ierr = PetscStrncmp(line, "$Nodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr); if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr); snum = sscanf(line, "%d", &numVertices); if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); ierr = PetscMalloc1(numVertices*3, &coordsIn);CHKERRQ(ierr); if (binary) { size_t doubleSize, intSize; PetscInt elementSize; char *buffer; PetscScalar *baseptr; ierr = PetscDataTypeGetSize(PETSC_ENUM, &intSize);CHKERRQ(ierr); ierr = PetscDataTypeGetSize(PETSC_DOUBLE, &doubleSize);CHKERRQ(ierr); elementSize = (intSize + 3*doubleSize); ierr = PetscMalloc1(elementSize*numVertices, &buffer);CHKERRQ(ierr); ierr = PetscViewerRead(viewer, buffer, elementSize*numVertices, NULL, PETSC_CHAR);CHKERRQ(ierr); if (bswap) ierr = PetscByteSwap(buffer, PETSC_CHAR, elementSize*numVertices);CHKERRQ(ierr); for (v = 0; v < numVertices; ++v) { baseptr = ((PetscScalar*)(buffer+v*elementSize+intSize)); coordsIn[v*3+0] = baseptr[0]; coordsIn[v*3+1] = baseptr[1]; coordsIn[v*3+2] = baseptr[2]; } ierr = PetscFree(buffer);CHKERRQ(ierr); } else { for (v = 0; v < numVertices; ++v) { ierr = PetscViewerRead(viewer, &i, 1, NULL, PETSC_ENUM);CHKERRQ(ierr); ierr = PetscViewerRead(viewer, &(coordsIn[v*3]), 3, NULL, PETSC_DOUBLE);CHKERRQ(ierr); if (i != (int)v+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid node number %d should be %d", i, v+1); } } ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);; ierr = PetscStrncmp(line, "$EndNodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr); if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); /* Read cells */ ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);; ierr = PetscStrncmp(line, "$Elements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr); if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);; snum = sscanf(line, "%d", &numCells); if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); } if (!rank || binary) { /* Gmsh elements can be of any dimension/co-dimension, so we need to traverse the file contents multiple times to figure out the true number of cells and facets in the given mesh. To make this more efficient we read the file contents only once and store them in memory, while determining the true number of cells. */ ierr = DMPlexCreateGmsh_ReadElement(viewer, numCells, binary, bswap, &gmsh_elem);CHKERRQ(ierr); for (trueNumCells=0, c = 0; c < numCells; ++c) { if (gmsh_elem[c].dim > dim) {dim = gmsh_elem[c].dim; trueNumCells = 0;} if (gmsh_elem[c].dim == dim) trueNumCells++; } ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);; ierr = PetscStrncmp(line, "$EndElements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr); if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file"); } /* For binary we read on all ranks, but only build the plex on rank 0 */ if (binary && rank) {trueNumCells = 0; numVertices = 0;}; /* Allocate the cell-vertex mesh */ ierr = DMPlexSetChart(*dm, 0, trueNumCells+numVertices);CHKERRQ(ierr); if (!rank) { for (cell = 0, c = 0; c < numCells; ++c) { if (gmsh_elem[c].dim == dim) { ierr = DMPlexSetConeSize(*dm, cell, gmsh_elem[c].numNodes);CHKERRQ(ierr); cell++; } } } ierr = DMSetUp(*dm);CHKERRQ(ierr); /* Add cell-vertex connections */ if (!rank) { PetscInt pcone[8], corner; for (cell = 0, c = 0; c < numCells; ++c) { if (gmsh_elem[c].dim == dim) { for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) { pcone[corner] = gmsh_elem[c].nodes[corner] + trueNumCells-1; } if (dim == 3) { /* Tetrahedra are inverted */ if (gmsh_elem[c].numNodes == 4) { PetscInt tmp = pcone[0]; pcone[0] = pcone[1]; pcone[1] = tmp; } /* Hexahedra are inverted */ if (gmsh_elem[c].numNodes == 8) { PetscInt tmp = pcone[1]; pcone[1] = pcone[3]; pcone[3] = tmp; } } ierr = DMPlexSetCone(*dm, cell, pcone);CHKERRQ(ierr); cell++; } } } ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr); ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr); ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr); ierr = DMPlexStratify(*dm);CHKERRQ(ierr); if (interpolate) { DM idm = NULL; ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr); ierr = DMDestroy(dm);CHKERRQ(ierr); *dm = idm; } if (!rank) { /* Apply boundary IDs by finding the relevant facets with vertex joins */ PetscInt pcone[8], corner, vStart, vEnd; ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr); for (c = 0; c < numCells; ++c) { if (gmsh_elem[c].dim == dim-1) { PetscInt joinSize; const PetscInt *join; for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) { pcone[corner] = gmsh_elem[c].nodes[corner] + vStart - 1; } ierr = DMPlexGetFullJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr); if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for element %d", gmsh_elem[c].id); ierr = DMSetLabelValue(*dm, "Face Sets", join[0], gmsh_elem[c].tags[0]);CHKERRQ(ierr); ierr = DMPlexRestoreJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr); } } /* Create cell sets */ for (cell = 0, c = 0; c < numCells; ++c) { if (gmsh_elem[c].dim == dim) { if (gmsh_elem[c].numTags > 0) { ierr = DMSetLabelValue(*dm, "Cell Sets", cell, gmsh_elem[c].tags[0]);CHKERRQ(ierr); cell++; } } } } /* Read coordinates */ ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr); ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr); ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr); ierr = PetscSectionSetChart(coordSection, trueNumCells, trueNumCells + numVertices);CHKERRQ(ierr); for (v = trueNumCells; v < trueNumCells+numVertices; ++v) { ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr); ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr); } ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr); ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr); ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr); ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr); ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr); ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr); ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr); if (!rank) { for (v = 0; v < numVertices; ++v) { for (d = 0; d < dim; ++d) { coords[v*dim+d] = coordsIn[v*3+d]; } } } ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr); ierr = PetscFree(coordsIn);CHKERRQ(ierr); ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr); ierr = VecDestroy(&coordinates);CHKERRQ(ierr); /* Clean up intermediate storage */ if (!rank || binary) ierr = PetscFree(gmsh_elem);CHKERRQ(ierr); ierr = PetscLogEventEnd(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **args) { Mat A,RHS,C,F,X; Vec u,x,b; PetscErrorCode ierr; PetscMPIInt rank,nproc; PetscInt i,m,n,nfact,nsolve,nrhs,ipack=0; PetscScalar *array,rval; PetscReal norm,tol=1.e-12; IS perm,iperm; MatFactorInfo info; PetscRandom rand; PetscBool flg,testMatSolve=PETSC_TRUE,testMatMatSolve=PETSC_TRUE; PetscViewer fd; /* viewer */ char file[PETSC_MAX_PATH_LEN]; /* input file name */ PetscInitialize(&argc,&args,(char*)0,help); ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD, &nproc);CHKERRQ(ierr); /* Determine file from which we read the matrix A */ ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_WORLD,1,"Must indicate binary file with the -f option"); /* Load matrix A */ ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatLoad(A,fd);CHKERRQ(ierr); ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr); ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr); if (m != n) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ, "This example is not intended for rectangular matrices (%d, %d)", m, n); /* Create dense matrix C and X; C holds true solution with identical colums */ nrhs = 2; ierr = PetscOptionsGetInt(NULL,"-nrhs",&nrhs,NULL);CHKERRQ(ierr); if (!rank) printf("ex125: nrhs %d\n",nrhs); ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr); ierr = MatSetSizes(C,m,PETSC_DECIDE,PETSC_DECIDE,nrhs);CHKERRQ(ierr); ierr = MatSetType(C,MATDENSE);CHKERRQ(ierr); ierr = MatSetFromOptions(C);CHKERRQ(ierr); ierr = MatSetUp(C);CHKERRQ(ierr); ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr); ierr = MatSetRandom(C,rand);CHKERRQ(ierr); ierr = MatDuplicate(C,MAT_DO_NOT_COPY_VALUES,&X);CHKERRQ(ierr); /* Create vectors */ ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr); ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(x);CHKERRQ(ierr); ierr = VecDuplicate(x,&b);CHKERRQ(ierr); ierr = VecDuplicate(x,&u);CHKERRQ(ierr); /* save the true solution */ /* Test LU Factorization */ ierr = MatGetOrdering(A,MATORDERINGND,&perm,&iperm);CHKERRQ(ierr); /*ierr = ISView(perm,PETSC_VIEWER_STDOUT_WORLD);*/ /*ierr = ISView(perm,PETSC_VIEWER_STDOUT_SELF);*/ ierr = PetscOptionsGetInt(NULL,"-mat_solver_package",&ipack,NULL);CHKERRQ(ierr); switch (ipack) { case 0: #if defined(PETSC_HAVE_SUPERLU) if (!rank) printf(" SUPERLU LU:\n"); ierr = MatGetFactor(A,MATSOLVERSUPERLU,MAT_FACTOR_LU,&F);CHKERRQ(ierr); break; #endif case 1: #if defined(PETSC_HAVE_SUPERLU_DIST) if (!rank) printf(" SUPERLU_DIST LU:\n"); ierr = MatGetFactor(A,MATSOLVERSUPERLU_DIST,MAT_FACTOR_LU,&F);CHKERRQ(ierr); break; #endif case 2: #if defined(PETSC_HAVE_MUMPS) if (!rank) printf(" MUMPS LU:\n"); ierr = MatGetFactor(A,MATSOLVERMUMPS,MAT_FACTOR_LU,&F);CHKERRQ(ierr); { /* test mumps options */ PetscInt icntl_7 = 5; ierr = MatMumpsSetIcntl(F,7,icntl_7);CHKERRQ(ierr); } break; #endif default: if (!rank) printf(" PETSC LU:\n"); ierr = MatGetFactor(A,MATSOLVERPETSC,MAT_FACTOR_LU,&F);CHKERRQ(ierr); } info.fill = 5.0; ierr = MatLUFactorSymbolic(F,A,perm,iperm,&info);CHKERRQ(ierr); for (nfact = 0; nfact < 2; nfact++) { if (!rank) printf(" %d-the LU numfactorization \n",nfact); ierr = MatLUFactorNumeric(F,A,&info);CHKERRQ(ierr); /* Test MatMatSolve() */ /* if ((ipack == 0 || ipack == 2) && testMatMatSolve) { printf(" MatMatSolve() is not implemented for this package. Skip the testing.\n"); testMatMatSolve = PETSC_FALSE; } */ if (testMatMatSolve) { if (!nfact) { ierr = MatMatMult(A,C,MAT_INITIAL_MATRIX,2.0,&RHS);CHKERRQ(ierr); } else { ierr = MatMatMult(A,C,MAT_REUSE_MATRIX,2.0,&RHS);CHKERRQ(ierr); } for (nsolve = 0; nsolve < 2; nsolve++) { if (!rank) printf(" %d-the MatMatSolve \n",nsolve); ierr = MatMatSolve(F,RHS,X);CHKERRQ(ierr); /* Check the error */ ierr = MatAXPY(X,-1.0,C,SAME_NONZERO_PATTERN);CHKERRQ(ierr); ierr = MatNorm(X,NORM_FROBENIUS,&norm);CHKERRQ(ierr); if (norm > tol) { if (!rank) { ierr = PetscPrintf(PETSC_COMM_SELF,"1st MatMatSolve: Norm of error %g, nsolve %d\n",norm,nsolve);CHKERRQ(ierr); } } } } /* Test MatSolve() */ if (testMatSolve) { for (nsolve = 0; nsolve < 2; nsolve++) { ierr = VecGetArray(x,&array);CHKERRQ(ierr); for (i=0; i<m; i++) { ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr); array[i] = rval; } ierr = VecRestoreArray(x,&array);CHKERRQ(ierr); ierr = VecCopy(x,u);CHKERRQ(ierr); ierr = MatMult(A,x,b);CHKERRQ(ierr); if (!rank) printf(" %d-the MatSolve \n",nsolve); ierr = MatSolve(F,b,x);CHKERRQ(ierr); /* Check the error */ ierr = VecAXPY(u,-1.0,x);CHKERRQ(ierr); /* u <- (-1.0)x + u */ ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr); if (norm > tol) { ierr = MatMult(A,x,u);CHKERRQ(ierr); /* u = A*x */ PetscReal resi; ierr = VecAXPY(u,-1.0,b);CHKERRQ(ierr); /* u <- (-1.0)b + u */ ierr = VecNorm(u,NORM_2,&resi);CHKERRQ(ierr); if (!rank) { ierr = PetscPrintf(PETSC_COMM_SELF,"MatSolve: Norm of error %g, resi %g, LU numfact %d\n",norm,resi,nfact);CHKERRQ(ierr); } } } } } /* Free data structures */ ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = MatDestroy(&F);CHKERRQ(ierr); ierr = MatDestroy(&X);CHKERRQ(ierr); if (testMatMatSolve) { ierr = MatDestroy(&RHS);CHKERRQ(ierr); } ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr); ierr = ISDestroy(&perm);CHKERRQ(ierr); ierr = ISDestroy(&iperm);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
int main(int argc, char** argv) { /* the total number of grid points in each spatial direction is (n+1) */ /* the total number of degrees-of-freedom in each spatial direction is (n-1) */ /* this version requires n to be a power of 2 */ if( argc < 2 ) { printf("need a problem size\n"); return 1; } int n = atoi(argv[1]); int m = n-1; // Initialize Petsc PetscInitialize(&argc,&argv,0,PETSC_NULL); // Create our vector Vec b; VecCreate(PETSC_COMM_WORLD,&b); VecSetSizes(b,PETSC_DECIDE,m*m); VecSetFromOptions(b); // Create our matrix Mat A; MatCreate(PETSC_COMM_WORLD,&A); MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,m*m,m*m); MatSetUp(A); // Create linear solver object KSP ksp; KSPCreate(PETSC_COMM_WORLD,&ksp); // setup rhs double h = 1./(double)n; for (int j=0; j < m*m; j++) VecSetValue(b,j,h*h,INSERT_VALUES); // setup matrix for (int i=0;i<m*m;++i) { MatSetValue(A,i,i,4.f,INSERT_VALUES); if (i%m != m-1) MatSetValue(A,i,i+1,-1.f,INSERT_VALUES); if (i%m) MatSetValue(A,i,i-1,-1.f,INSERT_VALUES); if (i > m) MatSetValue(A,i,i-m,-1.f,INSERT_VALUES); if (i < m*(m-1)) MatSetValue(A,i,i+m,-1.f,INSERT_VALUES); } // sync processes VecAssemblyBegin(b); VecAssemblyEnd(b); MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY); MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY); // solve KSPSetType(ksp,"cg"); KSPSetTolerances(ksp,1.e-10,1.e-10,1.e6,10000); KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN); PC pc; KSPGetPC(ksp,&pc); PCSetFromOptions(pc); PCSetUp(pc); // setup solver KSPSetFromOptions(ksp); KSPSetUp(ksp); Vec x; VecDuplicate(b,&x); KSPSolve(ksp,b,x); double val; VecNorm(x,NORM_INFINITY,&val); printf (" umax = %e \n",val); PetscFinalize(); return 0; }
int main(int argc,char **args) { Vec x,b,u; Mat A; /* linear system matrix */ KSP ksp; /* linear solver context */ PetscRandom rctx; /* random number generator context */ PetscReal norm; /* norm of solution error */ PetscInt i,j,k,l,n = 27,its,bs = 2,Ii,J; PetscErrorCode ierr; PetscScalar v; ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr; ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,n*bs,n*bs,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr); ierr = MatSetBlockSize(A,bs);CHKERRQ(ierr); ierr = MatSetType(A,MATSEQBAIJ);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); ierr = MatSetUp(A);CHKERRQ(ierr); /* Don't bother to preallocate matrix */ ierr = PetscRandomCreate(PETSC_COMM_SELF,&rctx);CHKERRQ(ierr); for (i=0; i<n; i++) { for (j=0; j<n; j++) { ierr = PetscRandomGetValue(rctx,&v);CHKERRQ(ierr); if (PetscRealPart(v) < .25 || i == j) { for (k=0; k<bs; k++) { for (l=0; l<bs; l++) { ierr = PetscRandomGetValue(rctx,&v);CHKERRQ(ierr); Ii = i*bs + k; J = j*bs + l; if (Ii == J) v += 10.; ierr = MatSetValue(A,Ii,J,v,INSERT_VALUES);CHKERRQ(ierr); } } } } } ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr); ierr = VecSetSizes(u,PETSC_DECIDE,n*bs);CHKERRQ(ierr); ierr = VecSetFromOptions(u);CHKERRQ(ierr); ierr = VecDuplicate(u,&b);CHKERRQ(ierr); ierr = VecDuplicate(b,&x);CHKERRQ(ierr); ierr = VecSet(u,1.0);CHKERRQ(ierr); ierr = MatMult(A,u,b);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create the linear solver and set various options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create linear solver context */ ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr); /* Set operators. Here the matrix that defines the linear system also serves as the preconditioning matrix. */ ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr); ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Solve the linear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Check solution and clean up - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Check the error */ ierr = VecAXPY(x,-1.0,u);CHKERRQ(ierr); ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr); ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr); /* Print convergence information. PetscPrintf() produces a single print statement from all processes that share a communicator. An alternative is PetscFPrintf(), which prints to a file. */ if (norm > .1) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of residual %g iterations %D bs %D\n",(double)norm,its,bs);CHKERRQ(ierr); } /* Free work space. All PETSc objects should be destroyed when they are no longer needed. */ ierr = KSPDestroy(&ksp);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr); /* Always call PetscFinalize() before exiting a program. This routine - finalizes the PETSc libraries as well as MPI - provides summary and diagnostic information if certain runtime options are chosen (e.g., -log_summary). */ ierr = PetscFinalize(); return ierr; }
int main(int argc,char **args) { Vec x,b,u; /* approx solution, RHS, exact solution */ Mat A; /* linear system matrix */ KSP ksp; /* linear solver context */ PetscReal norm; /* norm of solution error */ PetscInt dim,i,j,Ii,J,Istart,Iend,n = 6,its,use_random; PetscErrorCode ierr; PetscScalar v,none = -1.0,sigma2,pfive = 0.5,*xa; PetscRandom rctx; PetscReal h2,sigma1 = 100.0; PetscBool flg = PETSC_FALSE; ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr; #if !defined(PETSC_USE_COMPLEX) SETERRQ(PETSC_COMM_WORLD,1,"This example requires complex numbers"); #endif ierr = PetscOptionsGetReal(NULL,NULL,"-sigma1",&sigma1,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr); dim = n*n; /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Compute the matrix and right-hand-side vector that define the linear system, Ax = b. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create parallel matrix, specifying only its global dimensions. When using MatCreate(), the matrix format can be specified at runtime. Also, the parallel partitioning of the matrix is determined by PETSc at runtime. */ ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,dim,dim);CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); ierr = MatSetUp(A);CHKERRQ(ierr); /* Currently, all PETSc parallel matrix formats are partitioned by contiguous chunks of rows across the processors. Determine which rows of the matrix are locally owned. */ ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); /* Set matrix elements in parallel. - Each processor needs to insert only elements that it owns locally (but any non-local elements will be sent to the appropriate processor during matrix assembly). - Always specify global rows and columns of matrix entries. */ ierr = PetscOptionsGetBool(NULL,NULL,"-norandom",&flg,NULL);CHKERRQ(ierr); if (flg) use_random = 0; else use_random = 1; if (use_random) { ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr); ierr = PetscRandomSetInterval(rctx,0.0,PETSC_i);CHKERRQ(ierr); } else { sigma2 = 10.0*PETSC_i; } h2 = 1.0/((n+1)*(n+1)); for (Ii=Istart; Ii<Iend; Ii++) { v = -1.0; i = Ii/n; j = Ii - i*n; if (i>0) { J = Ii-n; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr); } if (i<n-1) { J = Ii+n; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr); } if (j>0) { J = Ii-1; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr); } if (j<n-1) { J = Ii+1; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr); } if (use_random) {ierr = PetscRandomGetValue(rctx,&sigma2);CHKERRQ(ierr);} v = 4.0 - sigma1*h2 + sigma2*h2; ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr); } if (use_random) {ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);} /* Assemble matrix, using the 2-step process: MatAssemblyBegin(), MatAssemblyEnd() Computations can be done while messages are in transition by placing code between these two statements. */ ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* Create parallel vectors. - When using VecCreate(), VecSetSizes() and VecSetFromOptions(), we specify only the vector's global dimension; the parallel partitioning is determined at runtime. - Note: We form 1 vector from scratch and then duplicate as needed. */ ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr); ierr = VecSetSizes(u,PETSC_DECIDE,dim);CHKERRQ(ierr); ierr = VecSetFromOptions(u);CHKERRQ(ierr); ierr = VecDuplicate(u,&b);CHKERRQ(ierr); ierr = VecDuplicate(b,&x);CHKERRQ(ierr); /* Set exact solution; then compute right-hand-side vector. */ if (use_random) { ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr); ierr = VecSetRandom(u,rctx);CHKERRQ(ierr); } else { ierr = VecSet(u,pfive);CHKERRQ(ierr); } ierr = MatMult(A,u,b);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create the linear solver and set various options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Create linear solver context */ ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr); /* Set operators. Here the matrix that defines the linear system also serves as the preconditioning matrix. */ ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr); /* Set runtime options, e.g., -ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol> */ ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Solve the linear system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Check solution and clean up - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ /* Print the first 3 entries of x; this demonstrates extraction of the real and imaginary components of the complex vector, x. */ flg = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,NULL,"-print_x3",&flg,NULL);CHKERRQ(ierr); if (flg) { ierr = VecGetArray(x,&xa);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"The first three entries of x are:\n");CHKERRQ(ierr); for (i=0; i<3; i++) { ierr = PetscPrintf(PETSC_COMM_WORLD,"x[%D] = %g + %g i\n",i,(double)PetscRealPart(xa[i]),(double)PetscImaginaryPart(xa[i]));CHKERRQ(ierr); } ierr = VecRestoreArray(x,&xa);CHKERRQ(ierr); } /* Check the error */ ierr = VecAXPY(x,none,u);CHKERRQ(ierr); ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr); ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr); if (norm < 1.e-12) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error < 1.e-12 iterations %D\n",its);CHKERRQ(ierr); } else { ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %g iterations %D\n",(double)norm,its);CHKERRQ(ierr); } /* Free work space. All PETSc objects should be destroyed when they are no longer needed. */ ierr = KSPDestroy(&ksp);CHKERRQ(ierr); if (use_random) {ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);} ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
/*@ MatNullSpaceCreateRigidBody - create rigid body modes from coordinates Collective on Vec Input Argument: . coords - block of coordinates of each node, must have block size set Output Argument: . sp - the null space Level: advanced .seealso: MatNullSpaceCreate() @*/ PetscErrorCode MatNullSpaceCreateRigidBody(Vec coords,MatNullSpace *sp) { PetscErrorCode ierr; const PetscScalar *x; PetscScalar *v[6],dots[5]; Vec vec[6]; PetscInt n,N,dim,nmodes,i,j; PetscFunctionBegin; ierr = VecGetBlockSize(coords,&dim);CHKERRQ(ierr); ierr = VecGetLocalSize(coords,&n);CHKERRQ(ierr); ierr = VecGetSize(coords,&N);CHKERRQ(ierr); n /= dim; N /= dim; switch (dim) { case 1: ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)coords),PETSC_TRUE,0,NULL,sp);CHKERRQ(ierr); break; case 2: case 3: nmodes = (dim == 2) ? 3 : 6; ierr = VecCreate(PetscObjectComm((PetscObject)coords),&vec[0]);CHKERRQ(ierr); ierr = VecSetSizes(vec[0],dim*n,dim*N);CHKERRQ(ierr); ierr = VecSetBlockSize(vec[0],dim);CHKERRQ(ierr); ierr = VecSetUp(vec[0]);CHKERRQ(ierr); for (i=1; i<nmodes; i++) {ierr = VecDuplicate(vec[0],&vec[i]);CHKERRQ(ierr);} for (i=0; i<nmodes; i++) {ierr = VecGetArray(vec[i],&v[i]);CHKERRQ(ierr);} ierr = VecGetArrayRead(coords,&x);CHKERRQ(ierr); for (i=0; i<n; i++) { if (dim == 2) { v[0][i*2+0] = 1./N; v[0][i*2+1] = 0.; v[1][i*2+0] = 0.; v[1][i*2+1] = 1./N; /* Rotations */ v[2][i*2+0] = -x[i*2+1]; v[2][i*2+1] = x[i*2+0]; } else { v[0][i*3+0] = 1./N; v[0][i*3+1] = 0.; v[0][i*3+2] = 0.; v[1][i*3+0] = 0.; v[1][i*3+1] = 1./N; v[1][i*3+2] = 0.; v[2][i*3+0] = 0.; v[2][i*3+1] = 0.; v[2][i*3+2] = 1./N; v[3][i*3+0] = x[i*3+1]; v[3][i*3+1] = -x[i*3+0]; v[3][i*3+2] = 0.; v[4][i*3+0] = 0.; v[4][i*3+1] = -x[i*3+2]; v[4][i*3+2] = x[i*3+1]; v[5][i*3+0] = x[i*3+2]; v[5][i*3+1] = 0.; v[5][i*3+2] = -x[i*3+0]; } } for (i=0; i<nmodes; i++) {ierr = VecRestoreArray(vec[i],&v[i]);CHKERRQ(ierr);} ierr = VecRestoreArrayRead(coords,&x);CHKERRQ(ierr); for (i=dim; i<nmodes; i++) { /* Orthonormalize vec[i] against vec[0:i-1] */ ierr = VecMDot(vec[i],i,vec,dots);CHKERRQ(ierr); for (j=0; j<i; j++) dots[j] *= -1.; ierr = VecMAXPY(vec[i],i,dots,vec);CHKERRQ(ierr); ierr = VecNormalize(vec[i],NULL);CHKERRQ(ierr); } ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)coords),PETSC_FALSE,nmodes,vec,sp);CHKERRQ(ierr); for (i=0; i<nmodes; i++) {ierr = VecDestroy(&vec[i]);CHKERRQ(ierr);} } PetscFunctionReturn(0); }
int main(int argc,char **argv) { PetscErrorCode ierr; DM da,*subda; PetscInt i,dim=3; PetscMPIInt size,rank; Vec v; Vec slvec,sgvec; IS *ois,*iis; VecScatter oscata; VecScatter *iscat,*oscat,*gscat; DMDALocalInfo info; ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,"-dim",&dim,NULL);CHKERRQ(ierr); /* Create distributed array and get vectors */ ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); if (dim == 2) { ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,&da);CHKERRQ(ierr); } else if (dim == 3) { ierr = DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,-4,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr); } ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr); ierr = DMCreateDomainDecomposition(da,NULL,NULL,&iis,&ois,&subda);CHKERRQ(ierr); ierr = DMCreateDomainDecompositionScatters(da,1,subda,&iscat,&oscat,&gscat);CHKERRQ(ierr); { DMDALocalInfo subinfo; MatStencil lower,upper; IS patchis,subpatchis; Vec smallvec; Vec largevec; VecScatter patchscat; ierr = DMDAGetLocalInfo(subda[0],&subinfo);CHKERRQ(ierr); lower.i = info.xs; lower.j = info.ys; lower.k = info.zs; upper.i = info.xs+info.xm; upper.j = info.ys+info.ym; upper.k = info.zs+info.zm; /* test the patch IS as a thing to scatter to/from */ ierr = DMDACreatePatchIS(da,&lower,&upper,&patchis);CHKERRQ(ierr); ierr = DMGetGlobalVector(da,&largevec);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_SELF,&smallvec);CHKERRQ(ierr); ierr = VecSetSizes(smallvec,info.dof*(upper.i - lower.i)*(upper.j - lower.j)*(upper.k - lower.k),PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(smallvec);CHKERRQ(ierr); ierr = VecScatterCreate(smallvec,NULL,largevec,patchis,&patchscat);CHKERRQ(ierr); ierr = FillLocalSubdomain(subda[0],smallvec);CHKERRQ(ierr); ierr = VecSet(largevec,0);CHKERRQ(ierr); ierr = VecScatterBegin(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); for (i = 0; i < size; i++) { if (i == rank) { ierr = ISView(patchis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); ierr = VecScatterView(patchscat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); ierr = VecView(smallvec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); } ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); } ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); ierr = VecView(largevec,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = VecDestroy(&smallvec);CHKERRQ(ierr); ierr = DMRestoreGlobalVector(da,&largevec);CHKERRQ(ierr); ierr = ISDestroy(&patchis);CHKERRQ(ierr); ierr = VecScatterDestroy(&patchscat);CHKERRQ(ierr); } /* view the various parts */ { for (i = 0; i < size; i++) { if (i == rank) { ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr); ierr = DMView(subda[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); } ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); } ierr = DMGetLocalVector(subda[0],&slvec);CHKERRQ(ierr); ierr = DMGetGlobalVector(subda[0],&sgvec);CHKERRQ(ierr); ierr = DMGetGlobalVector(da,&v);CHKERRQ(ierr); /* test filling outer between the big DM and the small ones with the IS scatter*/ ierr = VecScatterCreate(v,ois[0],sgvec,NULL,&oscata);CHKERRQ(ierr); ierr = FillLocalSubdomain(subda[0],sgvec);CHKERRQ(ierr); ierr = VecScatterBegin(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); /* test the local-to-local scatter */ /* fill up the local subdomain and then add them together */ ierr = FillLocalSubdomain(da,v);CHKERRQ(ierr); ierr = VecScatterBegin(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* test ghost scattering backwards */ ierr = VecSet(v,0);CHKERRQ(ierr); ierr = VecScatterBegin(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* test overlap scattering backwards */ ierr = DMLocalToGlobalBegin(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr); ierr = DMLocalToGlobalEnd(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr); ierr = VecSet(v,0);CHKERRQ(ierr); ierr = VecScatterBegin(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* test interior scattering backwards */ ierr = VecSet(v,0);CHKERRQ(ierr); ierr = VecScatterBegin(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); /* test matrix allocation */ for (i = 0; i < size; i++) { if (i == rank) { Mat m; ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr); ierr = DMSetMatType(subda[0],MATAIJ);CHKERRQ(ierr); ierr = DMCreateMatrix(subda[0],&m);CHKERRQ(ierr); ierr = MatView(m,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr); ierr = MatDestroy(&m);CHKERRQ(ierr); } ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr); } ierr = DMRestoreLocalVector(subda[0],&slvec);CHKERRQ(ierr); ierr = DMRestoreGlobalVector(subda[0],&sgvec);CHKERRQ(ierr); ierr = DMRestoreGlobalVector(da,&v);CHKERRQ(ierr); } ierr = DMDestroy(&subda[0]);CHKERRQ(ierr); ierr = ISDestroy(&ois[0]);CHKERRQ(ierr); ierr = ISDestroy(&iis[0]);CHKERRQ(ierr); ierr = VecScatterDestroy(&iscat[0]);CHKERRQ(ierr); ierr = VecScatterDestroy(&oscat[0]);CHKERRQ(ierr); ierr = VecScatterDestroy(&gscat[0]);CHKERRQ(ierr); ierr = VecScatterDestroy(&oscata);CHKERRQ(ierr); ierr = PetscFree(iscat);CHKERRQ(ierr); ierr = PetscFree(oscat);CHKERRQ(ierr); ierr = PetscFree(gscat);CHKERRQ(ierr); ierr = PetscFree(oscata);CHKERRQ(ierr); ierr = PetscFree(subda);CHKERRQ(ierr); ierr = PetscFree(ois);CHKERRQ(ierr); ierr = PetscFree(iis);CHKERRQ(ierr); ierr = DMDestroy(&da);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
PETSC_STATIC_INLINE PetscErrorCode DMInterpolate_Hex_Private(DMInterpolationInfo ctx, DM dm, Vec xLocal, Vec v) { DM dmCoord; SNES snes; KSP ksp; PC pc; Vec coordsLocal, r, ref, real; Mat J; PetscScalar *a, *coords; PetscInt p; PetscErrorCode ierr; PetscFunctionBegin; ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr); ierr = DMGetCoordinateDM(dm, &dmCoord);CHKERRQ(ierr); ierr = SNESCreate(PETSC_COMM_SELF, &snes);CHKERRQ(ierr); ierr = SNESSetOptionsPrefix(snes, "hex_interp_");CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_SELF, &r);CHKERRQ(ierr); ierr = VecSetSizes(r, 3, 3);CHKERRQ(ierr); ierr = VecSetType(r,dm->vectype);CHKERRQ(ierr); ierr = VecDuplicate(r, &ref);CHKERRQ(ierr); ierr = VecDuplicate(r, &real);CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_SELF, &J);CHKERRQ(ierr); ierr = MatSetSizes(J, 3, 3, 3, 3);CHKERRQ(ierr); ierr = MatSetType(J, MATSEQDENSE);CHKERRQ(ierr); ierr = MatSetUp(J);CHKERRQ(ierr); ierr = SNESSetFunction(snes, r, HexMap_Private, NULL);CHKERRQ(ierr); ierr = SNESSetJacobian(snes, J, J, HexJacobian_Private, NULL);CHKERRQ(ierr); ierr = SNESGetKSP(snes, &ksp);CHKERRQ(ierr); ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr); ierr = PCSetType(pc, PCLU);CHKERRQ(ierr); ierr = SNESSetFromOptions(snes);CHKERRQ(ierr); ierr = VecGetArray(ctx->coords, &coords);CHKERRQ(ierr); ierr = VecGetArray(v, &a);CHKERRQ(ierr); for (p = 0; p < ctx->n; ++p) { PetscScalar *x = NULL, *vertices = NULL; PetscScalar *xi; PetscReal xir[3]; PetscInt c = ctx->cells[p], comp, coordSize, xSize; /* Can make this do all points at once */ ierr = DMPlexVecGetClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);CHKERRQ(ierr); if (8*3 != coordSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", coordSize, 8*3); ierr = DMPlexVecGetClosure(dm, NULL, xLocal, c, &xSize, &x);CHKERRQ(ierr); if (8*ctx->dof != xSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", xSize, 8*ctx->dof); ierr = SNESSetFunction(snes, NULL, NULL, (void*) vertices);CHKERRQ(ierr); ierr = SNESSetJacobian(snes, NULL, NULL, NULL, (void*) vertices);CHKERRQ(ierr); ierr = VecGetArray(real, &xi);CHKERRQ(ierr); xi[0] = coords[p*ctx->dim+0]; xi[1] = coords[p*ctx->dim+1]; xi[2] = coords[p*ctx->dim+2]; ierr = VecRestoreArray(real, &xi);CHKERRQ(ierr); ierr = SNESSolve(snes, real, ref);CHKERRQ(ierr); ierr = VecGetArray(ref, &xi);CHKERRQ(ierr); xir[0] = PetscRealPart(xi[0]); xir[1] = PetscRealPart(xi[1]); xir[2] = PetscRealPart(xi[2]); for (comp = 0; comp < ctx->dof; ++comp) { a[p*ctx->dof+comp] = x[0*ctx->dof+comp]*(1-xir[0])*(1-xir[1])*(1-xir[2]) + x[3*ctx->dof+comp]* xir[0]*(1-xir[1])*(1-xir[2]) + x[2*ctx->dof+comp]* xir[0]* xir[1]*(1-xir[2]) + x[1*ctx->dof+comp]*(1-xir[0])* xir[1]*(1-xir[2]) + x[4*ctx->dof+comp]*(1-xir[0])*(1-xir[1])* xir[2] + x[5*ctx->dof+comp]* xir[0]*(1-xir[1])* xir[2] + x[6*ctx->dof+comp]* xir[0]* xir[1]* xir[2] + x[7*ctx->dof+comp]*(1-xir[0])* xir[1]* xir[2]; } ierr = VecRestoreArray(ref, &xi);CHKERRQ(ierr); ierr = DMPlexVecRestoreClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);CHKERRQ(ierr); ierr = DMPlexVecRestoreClosure(dm, NULL, xLocal, c, &xSize, &x);CHKERRQ(ierr); } ierr = VecRestoreArray(v, &a);CHKERRQ(ierr); ierr = VecRestoreArray(ctx->coords, &coords);CHKERRQ(ierr); ierr = SNESDestroy(&snes);CHKERRQ(ierr); ierr = VecDestroy(&r);CHKERRQ(ierr); ierr = VecDestroy(&ref);CHKERRQ(ierr); ierr = VecDestroy(&real);CHKERRQ(ierr); ierr = MatDestroy(&J);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode MatFactorNumeric_PaStiX(Mat F,Mat A,const MatFactorInfo *info) { Mat_Pastix *lu =(Mat_Pastix*)(F)->spptr; Mat *tseq; PetscErrorCode ierr = 0; PetscInt icntl; PetscInt M=A->rmap->N; PetscBool valOnly,flg, isSym; Mat F_diag; IS is_iden; Vec b; IS isrow; PetscBool isSeqAIJ,isSeqSBAIJ,isMPIAIJ; PetscFunctionBegin; ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isSeqAIJ);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&isMPIAIJ);CHKERRQ(ierr); ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQSBAIJ,&isSeqSBAIJ);CHKERRQ(ierr); if (lu->matstruc == DIFFERENT_NONZERO_PATTERN){ (F)->ops->solve = MatSolve_PaStiX; /* Initialize a PASTIX instance */ ierr = MPI_Comm_dup(((PetscObject)A)->comm,&(lu->pastix_comm));CHKERRQ(ierr); ierr = MPI_Comm_rank(lu->pastix_comm, &lu->commRank); CHKERRQ(ierr); ierr = MPI_Comm_size(lu->pastix_comm, &lu->commSize); CHKERRQ(ierr); /* Set pastix options */ lu->iparm[IPARM_MODIFY_PARAMETER] = API_NO; lu->iparm[IPARM_START_TASK] = API_TASK_INIT; lu->iparm[IPARM_END_TASK] = API_TASK_INIT; lu->rhsnbr = 1; /* Call to set default pastix options */ PASTIX_CALL(&(lu->pastix_data), lu->pastix_comm, lu->n, lu->colptr, lu->row, (PastixScalar*)lu->val, lu->perm, lu->invp, (PastixScalar*)lu->rhs, lu->rhsnbr, lu->iparm, lu->dparm); ierr = PetscOptionsBegin(((PetscObject)A)->comm,((PetscObject)A)->prefix,"PaStiX Options","Mat");CHKERRQ(ierr); icntl=-1; lu->iparm[IPARM_VERBOSE] = API_VERBOSE_NOT; ierr = PetscOptionsInt("-mat_pastix_verbose","iparm[IPARM_VERBOSE] : level of printing (0 to 2)","None",lu->iparm[IPARM_VERBOSE],&icntl,&flg);CHKERRQ(ierr); if ((flg && icntl >= 0) || PetscLogPrintInfo) { lu->iparm[IPARM_VERBOSE] = icntl; } icntl=-1; ierr = PetscOptionsInt("-mat_pastix_threadnbr","iparm[IPARM_THREAD_NBR] : Number of thread by MPI node","None",lu->iparm[IPARM_THREAD_NBR],&icntl,PETSC_NULL);CHKERRQ(ierr); if ((flg && icntl > 0)) { lu->iparm[IPARM_THREAD_NBR] = icntl; } PetscOptionsEnd(); valOnly = PETSC_FALSE; } else { if (isSeqAIJ || isMPIAIJ) { ierr = PetscFree(lu->colptr);CHKERRQ(ierr); ierr = PetscFree(lu->row);CHKERRQ(ierr); ierr = PetscFree(lu->val);CHKERRQ(ierr); valOnly = PETSC_FALSE; } else valOnly = PETSC_TRUE; } lu->iparm[IPARM_MATRIX_VERIFICATION] = API_YES; /* convert mpi A to seq mat A */ ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr); ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&tseq);CHKERRQ(ierr); ierr = ISDestroy(&isrow);CHKERRQ(ierr); ierr = MatConvertToCSC(*tseq,valOnly, &lu->n, &lu->colptr, &lu->row, &lu->val);CHKERRQ(ierr); ierr = MatIsSymmetric(*tseq,0.0,&isSym);CHKERRQ(ierr); ierr = MatDestroyMatrices(1,&tseq);CHKERRQ(ierr); if (!lu->perm) { ierr = PetscMalloc((lu->n)*sizeof(PetscInt) ,&(lu->perm));CHKERRQ(ierr); ierr = PetscMalloc((lu->n)*sizeof(PetscInt) ,&(lu->invp));CHKERRQ(ierr); } if (isSym) { /* On symmetric matrix, LLT */ lu->iparm[IPARM_SYM] = API_SYM_YES; lu->iparm[IPARM_FACTORIZATION] = API_FACT_LDLT; } else { /* On unsymmetric matrix, LU */ lu->iparm[IPARM_SYM] = API_SYM_NO; lu->iparm[IPARM_FACTORIZATION] = API_FACT_LU; } /*----------------*/ if (lu->matstruc == DIFFERENT_NONZERO_PATTERN){ if (!(isSeqAIJ || isSeqSBAIJ)) { /* PaStiX only supports centralized rhs. Create scatter scat_rhs for repeated use in MatSolve() */ ierr = VecCreateSeq(PETSC_COMM_SELF,A->cmap->N,&lu->b_seq);CHKERRQ(ierr); ierr = ISCreateStride(PETSC_COMM_SELF,A->cmap->N,0,1,&is_iden);CHKERRQ(ierr); ierr = VecCreate(((PetscObject)A)->comm,&b);CHKERRQ(ierr); ierr = VecSetSizes(b,A->rmap->n,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(b);CHKERRQ(ierr); ierr = VecScatterCreate(b,is_iden,lu->b_seq,is_iden,&lu->scat_rhs);CHKERRQ(ierr); ierr = VecScatterCreate(lu->b_seq,is_iden,b,is_iden,&lu->scat_sol);CHKERRQ(ierr); ierr = ISDestroy(&is_iden);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); } lu->iparm[IPARM_START_TASK] = API_TASK_ORDERING; lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT; PASTIX_CALL(&(lu->pastix_data), lu->pastix_comm, lu->n, lu->colptr, lu->row, (PastixScalar*)lu->val, lu->perm, lu->invp, (PastixScalar*)lu->rhs, lu->rhsnbr, lu->iparm, lu->dparm); if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]); } else { lu->iparm[IPARM_START_TASK] = API_TASK_NUMFACT; lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT; PASTIX_CALL(&(lu->pastix_data), lu->pastix_comm, lu->n, lu->colptr, lu->row, (PastixScalar*)lu->val, lu->perm, lu->invp, (PastixScalar*)lu->rhs, lu->rhsnbr, lu->iparm, lu->dparm); if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]); } if (lu->commSize > 1){ if ((F)->factortype == MAT_FACTOR_LU){ F_diag = ((Mat_MPIAIJ *)(F)->data)->A; } else { F_diag = ((Mat_MPISBAIJ *)(F)->data)->A; } F_diag->assembled = PETSC_TRUE; } (F)->assembled = PETSC_TRUE; lu->matstruc = SAME_NONZERO_PATTERN; lu->CleanUpPastix = PETSC_TRUE; PetscFunctionReturn(0); }
PetscErrorCode DMInterpolationSetUp(DMInterpolationInfo ctx, DM dm, PetscBool redundantPoints) { MPI_Comm comm = ctx->comm; PetscScalar *a; PetscInt p, q, i; PetscMPIInt rank, size; PetscErrorCode ierr; Vec pointVec; IS cellIS; PetscLayout layout; PetscReal *globalPoints; PetscScalar *globalPointsScalar; const PetscInt *ranges; PetscMPIInt *counts, *displs; const PetscInt *foundCells; PetscMPIInt *foundProcs, *globalProcs; PetscInt n, N; PetscFunctionBegin; PetscValidHeaderSpecific(dm, DM_CLASSID, 1); ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr); if (ctx->dim < 0) SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set"); /* Locate points */ n = ctx->nInput; if (!redundantPoints) { ierr = PetscLayoutCreate(comm, &layout);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(layout, n);CHKERRQ(ierr); ierr = PetscLayoutSetUp(layout);CHKERRQ(ierr); ierr = PetscLayoutGetSize(layout, &N);CHKERRQ(ierr); /* Communicate all points to all processes */ ierr = PetscMalloc3(N*ctx->dim,&globalPoints,size,&counts,size,&displs);CHKERRQ(ierr); ierr = PetscLayoutGetRanges(layout, &ranges);CHKERRQ(ierr); for (p = 0; p < size; ++p) { counts[p] = (ranges[p+1] - ranges[p])*ctx->dim; displs[p] = ranges[p]*ctx->dim; } ierr = MPI_Allgatherv(ctx->points, n*ctx->dim, MPIU_REAL, globalPoints, counts, displs, MPIU_REAL, comm);CHKERRQ(ierr); } else { N = n; globalPoints = ctx->points; counts = displs = NULL; layout = NULL; } #if 0 ierr = PetscMalloc3(N,&foundCells,N,&foundProcs,N,&globalProcs);CHKERRQ(ierr); /* foundCells[p] = m->locatePoint(&globalPoints[p*ctx->dim]); */ #else #if defined(PETSC_USE_COMPLEX) ierr = PetscMalloc1(N,&globalPointsScalar);CHKERRQ(ierr); for (i=0; i<N; i++) globalPointsScalar[i] = globalPoints[i]; #else globalPointsScalar = globalPoints; #endif ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, ctx->dim, N*ctx->dim, globalPointsScalar, &pointVec);CHKERRQ(ierr); ierr = PetscMalloc2(N,&foundProcs,N,&globalProcs);CHKERRQ(ierr); ierr = DMLocatePoints(dm, pointVec, &cellIS);CHKERRQ(ierr); ierr = ISGetIndices(cellIS, &foundCells);CHKERRQ(ierr); #endif for (p = 0; p < N; ++p) { if (foundCells[p] >= 0) foundProcs[p] = rank; else foundProcs[p] = size; } /* Let the lowest rank process own each point */ ierr = MPI_Allreduce(foundProcs, globalProcs, N, MPI_INT, MPI_MIN, comm);CHKERRQ(ierr); ctx->n = 0; for (p = 0; p < N; ++p) { if (globalProcs[p] == size) SETERRQ4(comm, PETSC_ERR_PLIB, "Point %d: %g %g %g not located in mesh", p, globalPoints[p*ctx->dim+0], ctx->dim > 1 ? globalPoints[p*ctx->dim+1] : 0.0, ctx->dim > 2 ? globalPoints[p*ctx->dim+2] : 0.0); else if (globalProcs[p] == rank) ctx->n++; } /* Create coordinates vector and array of owned cells */ ierr = PetscMalloc1(ctx->n, &ctx->cells);CHKERRQ(ierr); ierr = VecCreate(comm, &ctx->coords);CHKERRQ(ierr); ierr = VecSetSizes(ctx->coords, ctx->n*ctx->dim, PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetBlockSize(ctx->coords, ctx->dim);CHKERRQ(ierr); ierr = VecSetType(ctx->coords,VECSTANDARD);CHKERRQ(ierr); ierr = VecGetArray(ctx->coords, &a);CHKERRQ(ierr); for (p = 0, q = 0, i = 0; p < N; ++p) { if (globalProcs[p] == rank) { PetscInt d; for (d = 0; d < ctx->dim; ++d, ++i) a[i] = globalPoints[p*ctx->dim+d]; ctx->cells[q++] = foundCells[p]; } } ierr = VecRestoreArray(ctx->coords, &a);CHKERRQ(ierr); #if 0 ierr = PetscFree3(foundCells,foundProcs,globalProcs);CHKERRQ(ierr); #else ierr = PetscFree2(foundProcs,globalProcs);CHKERRQ(ierr); ierr = ISRestoreIndices(cellIS, &foundCells);CHKERRQ(ierr); ierr = ISDestroy(&cellIS);CHKERRQ(ierr); ierr = VecDestroy(&pointVec);CHKERRQ(ierr); #endif if ((void*)globalPointsScalar != (void*)globalPoints) {ierr = PetscFree(globalPointsScalar);CHKERRQ(ierr);} if (!redundantPoints) {ierr = PetscFree3(globalPoints,counts,displs);CHKERRQ(ierr);} ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **args) { Mat A; /* linear system matrix */ PetscErrorCode ierr; PetscMPIInt rank=0; PetscBool flg; PetscViewer fd; /* viewer */ PetscViewer log; char file[PETSC_MAX_PATH_LEN]; char logfile[PETSC_MAX_PATH_LEN]; char lockfile[PETSC_MAX_PATH_LEN], tmpstr[PETSC_MAX_PATH_LEN], dirname[PETSC_MAX_PATH_LEN], matrix[PETSC_MAX_PATH_LEN]; char hash[20]; PetscLogDouble solveTime,endTime,startTime; PetscInt its; PetscReal norm; KSP ksp; // Linear solver context Vec b,x,u; // RHS, solution, vector for norm calculation PetscScalar one = 1.0; PetscInt m, n, i; FILE *lock; /* if (rank == 0) { printf("Command line arguments:\n"); for (i=0; i < argc; i++) printf("%d: %s\n", i, args[i]); } // Save args int argcount = argc; char **argv = (char**) malloc (argc*sizeof(char*)); for (i=0; i < argc; i++) { argv[i] = (char*) malloc(strlen(args[i]) + 1); strcpy(argv[i],args[i]); } MPI_Comm_rank(MPI_COMM_WORLD,&rank); */ PetscInitialize(&argc,&args,(char *)0,help); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = PetscOptionsGetString(PETSC_NULL,"-hash",hash,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr); if (!flg) { strcpy(hash,"nohash"); } ierr = PetscOptionsGetString(PETSC_NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr); if (!flg) { PetscPrintf(PETSC_COMM_WORLD,"Must indicate matrix file with the -f option"); } /* Create lock file */ if (rank == 0) { for (i = strlen(file); i> 0; i--) if (file[i] == '.') break; strncpy(tmpstr, file, i-1); for (i = strlen(tmpstr); i> 0; i--) if (file[i] == '/') break; strncpy(dirname, tmpstr, i); dirname[i] = '\0'; sprintf(lockfile,"%s/../timing/.%s.%s", dirname, basename(tmpstr), hash); sprintf(logfile,"%s/../timing/%s.%s.log", dirname, basename(tmpstr), hash); lock = fopen(lockfile, "w"); fprintf(lock, "%s\n", file); fclose(lock); } /* Read file */ ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr); // Create matrix ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatSetType(A,MATMPIAIJ); CHKERRQ(ierr); ierr = MatSetFromOptions(A);CHKERRQ(ierr); // Load matrix from file ierr = MatLoad(A,fd);CHKERRQ(ierr); ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr); ierr = MatGetSize(A, &m, &n); CHKERRQ(ierr); // Assemble matrix //ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); //ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); // Create RHS vector ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr); ierr = VecSetSizes(b,PETSC_DECIDE,n); CHKERRQ(ierr); ierr = VecSetFromOptions(b);CHKERRQ(ierr); ierr = VecSet(b,one); CHKERRQ(ierr); //ierr = VecLoad(b,fd);CHKERRQ(ierr); // Create vectors x and u ierr = VecDuplicate(b,&x);CHKERRQ(ierr); ierr = VecDuplicate(b,&u);CHKERRQ(ierr); // Create KSP ierr = KSPCreate(PETSC_COMM_WORLD,&ksp); CHKERRQ(ierr); ierr = KSPSetInitialGuessNonzero(ksp,PETSC_FALSE);CHKERRQ(ierr); ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr); ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr); // Setup KSP ierr = KSPSetUp(ksp);CHKERRQ(ierr); ierr = KSPSetUpOnBlocks(ksp);CHKERRQ(ierr); // Get start time ierr = PetscTime(&startTime);CHKERRQ(ierr); // Get KSP and PC type KSPType kt; ierr = KSPGetType(ksp,&kt); PC pc; ierr = KSPGetPC(ksp,&pc); PCType pt; ierr = PCGetType(pc,&pt); // Print method info ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, logfile, &log); CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(log, "Hash: %s\n", hash); ierr = PetscViewerASCIIPrintf(log, "%s | %s",kt,pt);CHKERRQ(ierr); // Make sure the program doesn't crash // while trying to solve the system PetscPushErrorHandler(PetscIgnoreErrorHandler,NULL); ierr = KSPSolve(ksp,b,x); PetscPopErrorHandler(); // Check if anything went wrong if(ierr == 0 || ierr == -1){ // If no error occurred or stopped by MyKSPMonitor, // compute normal and stuff ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr); ierr = MatMult(A,x,u);CHKERRQ(ierr); ierr = VecAXPY(u,-1.0,b);CHKERRQ(ierr); ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr); ierr = PetscTime(&endTime);CHKERRQ(ierr); // Compute solve time solveTime = endTime - startTime; // Check if KSP converged KSPConvergedReason reason; KSPGetConvergedReason(ksp,&reason); // Print convergence code, solve time, preconditioned norm, iterations ierr = PetscViewerASCIIPrintf(log, " | %D | %e | %g | %D\n",reason,solveTime,norm,its);CHKERRQ(ierr); ierr = KSPView(ksp,log); ierr = PCView(pc,log); ierr = PetscLogView(log); } else{ // Disaster happened, bail out if (rank == 0) remove(lockfile); PetscFinalize(); return 0; } // Again, destroy KSP and vector ierr = KSPDestroy(&ksp);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr); if (rank == 0) remove(lockfile); PetscFinalize(); return 0; }
/* Two triangles separated by a zero-volume cell with 6 vertices 5--16--8 / | | \ 11 | | 12 / | | \ 3 0 10 2 14 1 6 \ | | / 9 | | 13 \ | | / 4--15--7 */ PetscErrorCode CreateSimplexHybrid_2D(MPI_Comm comm, DM dm) { Vec coordinates; PetscSection coordSection; PetscScalar *coords; PetscInt numVertices = 0, numEdges = 0, numCells = 0, cMax = PETSC_DETERMINE, fMax = PETSC_DETERMINE; PetscInt firstVertex, firstEdge, coordSize; PetscInt v, e; PetscMPIInt rank; PetscErrorCode ierr; PetscFunctionBegin; ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr); if (!rank) { numVertices = 3 + 3; numEdges = 6 + 2; numCells = 3; cMax = 2; fMax = 15; } firstVertex = numCells; firstEdge = numCells + numVertices; ierr = DMPlexSetChart(dm, 0, numCells+numEdges+numVertices);CHKERRQ(ierr); if (numCells) { ierr = DMPlexSetConeSize(dm, 0, 3);CHKERRQ(ierr); ierr = DMPlexSetConeSize(dm, 1, 3);CHKERRQ(ierr); ierr = DMPlexSetConeSize(dm, 2, 4);CHKERRQ(ierr); } for(e = firstEdge; e < firstEdge+numEdges; ++e) { ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr); } ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */ ierr = DMPlexSetHybridBounds(dm, cMax, PETSC_DETERMINE, fMax, PETSC_DETERMINE);CHKERRQ(ierr); /* Indicate a hybrid mesh */ /* Build cells */ if (numCells > 0) { const PetscInt cone[3] = {9, 10, 11}; const PetscInt ornt[3] = {0, 0, 0}; const PetscInt cell = 0; ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr); ierr = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr); } if (numCells > 1) { const PetscInt cone[3] = {12, 14, 13}; const PetscInt ornt[3] = { 0, -2, 0}; const PetscInt cell = 1; ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr); ierr = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr); } if (numCells > 2) { const PetscInt cone[4] = {10, 14, 15, 16}; const PetscInt ornt[4] = { 0, 0, 0, 0}; const PetscInt cell = 2; ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr); ierr = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr); } /* Build edges*/ if (numEdges > 0) { const PetscInt cone[2] = {3, 4}; const PetscInt edge = 9; ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr); ierr = DMPlexSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr); } if (numEdges > 1) { const PetscInt cone[2] = {4, 5}; const PetscInt edge = 10; ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr); } if (numEdges > 2) { const PetscInt cone[2] = {5, 3}; const PetscInt edge = 11; ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr); ierr = DMPlexSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr); } if (numEdges > 3) { const PetscInt cone[2] = {6, 8}; const PetscInt edge = 12; ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr); ierr = DMPlexSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr); } if (numEdges > 4) { const PetscInt cone[2] = {7, 6}; const PetscInt edge = 13; ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr); ierr = DMPlexSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr); } if (numEdges > 5) { const PetscInt cone[2] = {7, 8}; const PetscInt edge = 14; ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr); } if (numEdges > 6) { const PetscInt cone[2] = {4, 7}; const PetscInt edge = 15; ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr); } if (numEdges > 7) { const PetscInt cone[2] = {5, 8}; const PetscInt edge = 16; ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr); } ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr); ierr = DMPlexStratify(dm);CHKERRQ(ierr); /* Build coordinates */ ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr); ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr); for(v = firstVertex; v < firstVertex+numVertices; ++v) { ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr); } ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr); ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr); ierr = VecCreate(((PetscObject) dm)->comm, &coordinates);CHKERRQ(ierr); ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr); ierr = VecSetFromOptions(coordinates);CHKERRQ(ierr); ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr); if (numVertices) { coords[0] = -0.5; coords[1] = 0.5; coords[2] = -0.2; coords[3] = 0.0; coords[4] = -0.2; coords[5] = 1.0; coords[6] = 0.5; coords[7] = 0.5; coords[8] = 0.2; coords[9] = 0.0; coords[10] = 0.2; coords[11] = 1.0; } ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr); ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr); ierr = VecDestroy(&coordinates);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode PCBDDCSetupFETIDPMatContext(FETIDPMat_ctx fetidpmat_ctx ) { PetscErrorCode ierr; PC_IS *pcis=(PC_IS*)fetidpmat_ctx->pc->data; PC_BDDC *pcbddc=(PC_BDDC*)fetidpmat_ctx->pc->data; PCBDDCGraph mat_graph=pcbddc->mat_graph; Mat_IS *matis = (Mat_IS*)fetidpmat_ctx->pc->pmat->data; MPI_Comm comm; Mat ScalingMat; Vec lambda_global; IS IS_l2g_lambda; IS subset,subset_mult,subset_n; PetscBool skip_node,fully_redundant; PetscInt i,j,k,s,n_boundary_dofs,n_global_lambda,n_vertices,partial_sum; PetscInt cum,n_local_lambda,n_lambda_for_dof,dual_size,n_neg_values,n_pos_values; PetscMPIInt rank,size,buf_size,neigh; PetscScalar scalar_value; PetscInt *vertex_indices; PetscInt *dual_dofs_boundary_indices,*aux_local_numbering_1; const PetscInt *aux_global_numbering; PetscInt *aux_sums,*cols_B_delta,*l2g_indices; PetscScalar *array,*scaling_factors,*vals_B_delta; PetscInt *aux_local_numbering_2; /* For communication of scaling factors */ PetscInt *ptrs_buffer,neigh_position; PetscScalar **all_factors,*send_buffer,*recv_buffer; MPI_Request *send_reqs,*recv_reqs; /* tests */ Vec test_vec; PetscBool test_fetidp; PetscViewer viewer; PetscFunctionBegin; ierr = PetscObjectGetComm((PetscObject)(fetidpmat_ctx->pc),&comm);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); /* Default type of lagrange multipliers is non-redundant */ fully_redundant = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,"-fetidp_fullyredundant",&fully_redundant,NULL);CHKERRQ(ierr); /* Evaluate local and global number of lagrange multipliers */ ierr = VecSet(pcis->vec1_N,0.0);CHKERRQ(ierr); n_local_lambda = 0; partial_sum = 0; n_boundary_dofs = 0; s = 0; /* Get Vertices used to define the BDDC */ n_vertices = pcbddc->n_vertices; vertex_indices = pcbddc->local_primal_ref_node; dual_size = pcis->n_B-n_vertices; ierr = PetscMalloc1(dual_size,&dual_dofs_boundary_indices);CHKERRQ(ierr); ierr = PetscMalloc1(dual_size,&aux_local_numbering_1);CHKERRQ(ierr); ierr = PetscMalloc1(dual_size,&aux_local_numbering_2);CHKERRQ(ierr); ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr); for (i=0;i<pcis->n;i++){ j = mat_graph->count[i]; /* RECALL: mat_graph->count[i] does not count myself */ if ( j > 0 ) { n_boundary_dofs++; } skip_node = PETSC_FALSE; if ( s < n_vertices && vertex_indices[s]==i) { /* it works for a sorted set of vertices */ skip_node = PETSC_TRUE; s++; } if (j < 1) { skip_node = PETSC_TRUE; } if ( !skip_node ) { if (fully_redundant) { /* fully redundant set of lagrange multipliers */ n_lambda_for_dof = (j*(j+1))/2; } else { n_lambda_for_dof = j; } n_local_lambda += j; /* needed to evaluate global number of lagrange multipliers */ array[i]=(1.0*n_lambda_for_dof)/(j+1.0); /* already scaled for the next global sum */ /* store some data needed */ dual_dofs_boundary_indices[partial_sum] = n_boundary_dofs-1; aux_local_numbering_1[partial_sum] = i; aux_local_numbering_2[partial_sum] = n_lambda_for_dof; partial_sum++; } } ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr); ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr); ierr = VecScatterBegin(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecSum(pcis->vec1_global,&scalar_value);CHKERRQ(ierr); fetidpmat_ctx->n_lambda = (PetscInt)PetscRealPart(scalar_value); /* compute global ordering of lagrange multipliers and associate l2g map */ ierr = ISCreateGeneral(comm,partial_sum,aux_local_numbering_1,PETSC_COPY_VALUES,&subset_n);CHKERRQ(ierr); ierr = ISLocalToGlobalMappingApplyIS(pcis->mapping,subset_n,&subset);CHKERRQ(ierr); ierr = ISDestroy(&subset_n);CHKERRQ(ierr); ierr = ISCreateGeneral(comm,partial_sum,aux_local_numbering_2,PETSC_OWN_POINTER,&subset_mult);CHKERRQ(ierr); ierr = PCBDDCSubsetNumbering(subset,subset_mult,&i,&subset_n);CHKERRQ(ierr); ierr = ISDestroy(&subset);CHKERRQ(ierr); if (i != fetidpmat_ctx->n_lambda) { SETERRQ3(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"Error in %s: global number of multipliers mismatch! (%d!=%d)\n",__FUNCT__,fetidpmat_ctx->n_lambda,i); } /* init data for scaling factors exchange */ partial_sum = 0; j = 0; ierr = PetscMalloc1(pcis->n_neigh,&ptrs_buffer);CHKERRQ(ierr); ierr = PetscMalloc1(pcis->n_neigh-1,&send_reqs);CHKERRQ(ierr); ierr = PetscMalloc1(pcis->n_neigh-1,&recv_reqs);CHKERRQ(ierr); ierr = PetscMalloc1(pcis->n,&all_factors);CHKERRQ(ierr); ptrs_buffer[0]=0; for (i=1;i<pcis->n_neigh;i++) { partial_sum += pcis->n_shared[i]; ptrs_buffer[i] = ptrs_buffer[i-1]+pcis->n_shared[i]; } ierr = PetscMalloc1(partial_sum,&send_buffer);CHKERRQ(ierr); ierr = PetscMalloc1(partial_sum,&recv_buffer);CHKERRQ(ierr); ierr = PetscMalloc1(partial_sum,&all_factors[0]);CHKERRQ(ierr); for (i=0;i<pcis->n-1;i++) { j = mat_graph->count[i]; all_factors[i+1]=all_factors[i]+j; } /* scatter B scaling to N vec */ ierr = VecScatterBegin(pcis->N_to_B,pcis->D,pcis->vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(pcis->N_to_B,pcis->D,pcis->vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); /* communications */ ierr = VecGetArrayRead(pcis->vec1_N,(const PetscScalar**)&array);CHKERRQ(ierr); for (i=1;i<pcis->n_neigh;i++) { for (j=0;j<pcis->n_shared[i];j++) { send_buffer[ptrs_buffer[i-1]+j]=array[pcis->shared[i][j]]; } ierr = PetscMPIIntCast(ptrs_buffer[i]-ptrs_buffer[i-1],&buf_size);CHKERRQ(ierr); ierr = PetscMPIIntCast(pcis->neigh[i],&neigh);CHKERRQ(ierr); ierr = MPI_Isend(&send_buffer[ptrs_buffer[i-1]],buf_size,MPIU_SCALAR,neigh,0,comm,&send_reqs[i-1]);CHKERRQ(ierr); ierr = MPI_Irecv(&recv_buffer[ptrs_buffer[i-1]],buf_size,MPIU_SCALAR,neigh,0,comm,&recv_reqs[i-1]);CHKERRQ(ierr); } ierr = VecRestoreArrayRead(pcis->vec1_N,(const PetscScalar**)&array);CHKERRQ(ierr); ierr = MPI_Waitall((pcis->n_neigh-1),recv_reqs,MPI_STATUSES_IGNORE);CHKERRQ(ierr); /* put values in correct places */ for (i=1;i<pcis->n_neigh;i++) { for (j=0;j<pcis->n_shared[i];j++) { k = pcis->shared[i][j]; neigh_position = 0; while(mat_graph->neighbours_set[k][neigh_position] != pcis->neigh[i]) {neigh_position++;} all_factors[k][neigh_position]=recv_buffer[ptrs_buffer[i-1]+j]; } } ierr = MPI_Waitall((pcis->n_neigh-1),send_reqs,MPI_STATUSES_IGNORE);CHKERRQ(ierr); ierr = PetscFree(send_reqs);CHKERRQ(ierr); ierr = PetscFree(recv_reqs);CHKERRQ(ierr); ierr = PetscFree(send_buffer);CHKERRQ(ierr); ierr = PetscFree(recv_buffer);CHKERRQ(ierr); ierr = PetscFree(ptrs_buffer);CHKERRQ(ierr); /* Compute B and B_delta (local actions) */ ierr = PetscMalloc1(pcis->n_neigh,&aux_sums);CHKERRQ(ierr); ierr = PetscMalloc1(n_local_lambda,&l2g_indices);CHKERRQ(ierr); ierr = PetscMalloc1(n_local_lambda,&vals_B_delta);CHKERRQ(ierr); ierr = PetscMalloc1(n_local_lambda,&cols_B_delta);CHKERRQ(ierr); ierr = PetscMalloc1(n_local_lambda,&scaling_factors);CHKERRQ(ierr); ierr = ISGetIndices(subset_n,&aux_global_numbering);CHKERRQ(ierr); n_global_lambda=0; partial_sum=0; cum = 0; for (i=0;i<dual_size;i++) { n_global_lambda = aux_global_numbering[cum]; j = mat_graph->count[aux_local_numbering_1[i]]; aux_sums[0]=0; for (s=1;s<j;s++) { aux_sums[s]=aux_sums[s-1]+j-s+1; } array = all_factors[aux_local_numbering_1[i]]; n_neg_values = 0; while(n_neg_values < j && mat_graph->neighbours_set[aux_local_numbering_1[i]][n_neg_values] < rank) {n_neg_values++;} n_pos_values = j - n_neg_values; if (fully_redundant) { for (s=0;s<n_neg_values;s++) { l2g_indices [partial_sum+s]=aux_sums[s]+n_neg_values-s-1+n_global_lambda; cols_B_delta [partial_sum+s]=dual_dofs_boundary_indices[i]; vals_B_delta [partial_sum+s]=-1.0; scaling_factors[partial_sum+s]=array[s]; } for (s=0;s<n_pos_values;s++) { l2g_indices [partial_sum+s+n_neg_values]=aux_sums[n_neg_values]+s+n_global_lambda; cols_B_delta [partial_sum+s+n_neg_values]=dual_dofs_boundary_indices[i]; vals_B_delta [partial_sum+s+n_neg_values]=1.0; scaling_factors[partial_sum+s+n_neg_values]=array[s+n_neg_values]; } partial_sum += j; } else { /* l2g_indices and default cols and vals of B_delta */ for (s=0;s<j;s++) { l2g_indices [partial_sum+s]=n_global_lambda+s; cols_B_delta [partial_sum+s]=dual_dofs_boundary_indices[i]; vals_B_delta [partial_sum+s]=0.0; } /* B_delta */ if ( n_neg_values > 0 ) { /* there's a rank next to me to the left */ vals_B_delta [partial_sum+n_neg_values-1]=-1.0; } if ( n_neg_values < j ) { /* there's a rank next to me to the right */ vals_B_delta [partial_sum+n_neg_values]=1.0; } /* scaling as in Klawonn-Widlund 1999*/ for (s=0;s<n_neg_values;s++) { scalar_value = 0.0; for (k=0;k<s+1;k++) { scalar_value += array[k]; } scaling_factors[partial_sum+s] = -scalar_value; } for (s=0;s<n_pos_values;s++) { scalar_value = 0.0; for (k=s+n_neg_values;k<j;k++) { scalar_value += array[k]; } scaling_factors[partial_sum+s+n_neg_values] = scalar_value; } partial_sum += j; } cum += aux_local_numbering_2[i]; } ierr = ISRestoreIndices(subset_n,&aux_global_numbering);CHKERRQ(ierr); ierr = ISDestroy(&subset_mult);CHKERRQ(ierr); ierr = ISDestroy(&subset_n);CHKERRQ(ierr); ierr = PetscFree(aux_sums);CHKERRQ(ierr); ierr = PetscFree(aux_local_numbering_1);CHKERRQ(ierr); ierr = PetscFree(dual_dofs_boundary_indices);CHKERRQ(ierr); ierr = PetscFree(all_factors[0]);CHKERRQ(ierr); ierr = PetscFree(all_factors);CHKERRQ(ierr); /* Local to global mapping of fetidpmat */ ierr = VecCreate(PETSC_COMM_SELF,&fetidpmat_ctx->lambda_local);CHKERRQ(ierr); ierr = VecSetSizes(fetidpmat_ctx->lambda_local,n_local_lambda,n_local_lambda);CHKERRQ(ierr); ierr = VecSetType(fetidpmat_ctx->lambda_local,VECSEQ);CHKERRQ(ierr); ierr = VecCreate(comm,&lambda_global);CHKERRQ(ierr); ierr = VecSetSizes(lambda_global,PETSC_DECIDE,fetidpmat_ctx->n_lambda);CHKERRQ(ierr); ierr = VecSetType(lambda_global,VECMPI);CHKERRQ(ierr); ierr = ISCreateGeneral(comm,n_local_lambda,l2g_indices,PETSC_OWN_POINTER,&IS_l2g_lambda);CHKERRQ(ierr); ierr = VecScatterCreate(fetidpmat_ctx->lambda_local,(IS)0,lambda_global,IS_l2g_lambda,&fetidpmat_ctx->l2g_lambda);CHKERRQ(ierr); ierr = ISDestroy(&IS_l2g_lambda);CHKERRQ(ierr); /* Create local part of B_delta */ ierr = MatCreate(PETSC_COMM_SELF,&fetidpmat_ctx->B_delta);CHKERRQ(ierr); ierr = MatSetSizes(fetidpmat_ctx->B_delta,n_local_lambda,pcis->n_B,n_local_lambda,pcis->n_B);CHKERRQ(ierr); ierr = MatSetType(fetidpmat_ctx->B_delta,MATSEQAIJ);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(fetidpmat_ctx->B_delta,1,NULL);CHKERRQ(ierr); ierr = MatSetOption(fetidpmat_ctx->B_delta,MAT_IGNORE_ZERO_ENTRIES,PETSC_TRUE);CHKERRQ(ierr); for (i=0;i<n_local_lambda;i++) { ierr = MatSetValue(fetidpmat_ctx->B_delta,i,cols_B_delta[i],vals_B_delta[i],INSERT_VALUES);CHKERRQ(ierr); } ierr = PetscFree(vals_B_delta);CHKERRQ(ierr); ierr = MatAssemblyBegin(fetidpmat_ctx->B_delta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd (fetidpmat_ctx->B_delta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); if (fully_redundant) { ierr = MatCreate(PETSC_COMM_SELF,&ScalingMat);CHKERRQ(ierr); ierr = MatSetSizes(ScalingMat,n_local_lambda,n_local_lambda,n_local_lambda,n_local_lambda);CHKERRQ(ierr); ierr = MatSetType(ScalingMat,MATSEQAIJ);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(ScalingMat,1,NULL);CHKERRQ(ierr); for (i=0;i<n_local_lambda;i++) { ierr = MatSetValue(ScalingMat,i,i,scaling_factors[i],INSERT_VALUES);CHKERRQ(ierr); } ierr = MatAssemblyBegin(ScalingMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd (ScalingMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatMatMult(ScalingMat,fetidpmat_ctx->B_delta,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&fetidpmat_ctx->B_Ddelta);CHKERRQ(ierr); ierr = MatDestroy(&ScalingMat);CHKERRQ(ierr); } else { ierr = MatCreate(PETSC_COMM_SELF,&fetidpmat_ctx->B_Ddelta);CHKERRQ(ierr); ierr = MatSetSizes(fetidpmat_ctx->B_Ddelta,n_local_lambda,pcis->n_B,n_local_lambda,pcis->n_B);CHKERRQ(ierr); ierr = MatSetType(fetidpmat_ctx->B_Ddelta,MATSEQAIJ);CHKERRQ(ierr); ierr = MatSeqAIJSetPreallocation(fetidpmat_ctx->B_Ddelta,1,NULL);CHKERRQ(ierr); for (i=0;i<n_local_lambda;i++) { ierr = MatSetValue(fetidpmat_ctx->B_Ddelta,i,cols_B_delta[i],scaling_factors[i],INSERT_VALUES);CHKERRQ(ierr); } ierr = MatAssemblyBegin(fetidpmat_ctx->B_Ddelta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd (fetidpmat_ctx->B_Ddelta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); } ierr = PetscFree(scaling_factors);CHKERRQ(ierr); ierr = PetscFree(cols_B_delta);CHKERRQ(ierr); /* Create some vectors needed by fetidp */ ierr = VecDuplicate(pcis->vec1_B,&fetidpmat_ctx->temp_solution_B);CHKERRQ(ierr); ierr = VecDuplicate(pcis->vec1_D,&fetidpmat_ctx->temp_solution_D);CHKERRQ(ierr); test_fetidp = PETSC_FALSE; ierr = PetscOptionsGetBool(NULL,"-fetidp_check",&test_fetidp,NULL);CHKERRQ(ierr); if (test_fetidp && !pcbddc->use_deluxe_scaling) { PetscReal real_value; ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr); ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(viewer,"----------FETI_DP TESTS--------------\n");CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(viewer,"All tests should return zero!\n");CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(viewer,"FETIDP MAT context in the ");CHKERRQ(ierr); if (fully_redundant) { ierr = PetscViewerASCIIPrintf(viewer,"fully redundant case for lagrange multipliers.\n");CHKERRQ(ierr); } else { ierr = PetscViewerASCIIPrintf(viewer,"Non-fully redundant case for lagrange multiplier.\n");CHKERRQ(ierr); } ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); /******************************************************************/ /* TEST A/B: Test numbering of global lambda dofs */ /******************************************************************/ ierr = VecDuplicate(fetidpmat_ctx->lambda_local,&test_vec);CHKERRQ(ierr); ierr = VecSet(lambda_global,1.0);CHKERRQ(ierr); ierr = VecSet(test_vec,1.0);CHKERRQ(ierr); ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); scalar_value = -1.0; ierr = VecAXPY(test_vec,scalar_value,fetidpmat_ctx->lambda_local);CHKERRQ(ierr); ierr = VecNorm(test_vec,NORM_INFINITY,&real_value);CHKERRQ(ierr); ierr = VecDestroy(&test_vec);CHKERRQ(ierr); ierr = PetscViewerASCIISynchronizedPrintf(viewer,"A[%04d]: CHECK glob to loc: % 1.14e\n",rank,real_value);CHKERRQ(ierr); ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); if (fully_redundant) { ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr); ierr = VecSet(fetidpmat_ctx->lambda_local,0.5);CHKERRQ(ierr); ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecSum(lambda_global,&scalar_value);CHKERRQ(ierr); ierr = PetscViewerASCIISynchronizedPrintf(viewer,"B[%04d]: CHECK loc to glob: % 1.14e\n",rank,PetscRealPart(scalar_value)-fetidpmat_ctx->n_lambda);CHKERRQ(ierr); ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); } /******************************************************************/ /* TEST C: It should holds B_delta*w=0, w\in\widehat{W} */ /* This is the meaning of the B matrix */ /******************************************************************/ ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr); ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr); ierr = VecScatterBegin(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterBegin(matis->rctx,pcis->vec1_global,pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd(matis->rctx,pcis->vec1_global,pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); /* Action of B_delta */ ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr); ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr); ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecNorm(lambda_global,NORM_INFINITY,&real_value);CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(viewer,"C[coll]: CHECK infty norm of B_delta*w (w continuous): % 1.14e\n",real_value);CHKERRQ(ierr); ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); /******************************************************************/ /* TEST D: It should holds E_Dw = w - P_Dw w\in\widetilde{W} */ /* E_D = R_D^TR */ /* P_D = B_{D,delta}^T B_{delta} */ /* eq.44 Mandel Tezaur and Dohrmann 2005 */ /******************************************************************/ /* compute a random vector in \widetilde{W} */ ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr); scalar_value = 0.0; /* set zero at vertices */ ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr); for (i=0;i<n_vertices;i++) { array[vertex_indices[i]]=scalar_value; } ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr); /* store w for final comparison */ ierr = VecDuplicate(pcis->vec1_B,&test_vec);CHKERRQ(ierr); ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,test_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,test_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); /* Jump operator P_D : results stored in pcis->vec1_B */ ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); /* Action of B_delta */ ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr); ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr); ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); /* Action of B_Ddelta^T */ ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr); /* Average operator E_D : results stored in pcis->vec2_B */ ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = PCBDDCScalingExtension(fetidpmat_ctx->pc,pcis->vec2_B,pcis->vec1_global);CHKERRQ(ierr); ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_global,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_global,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); /* test E_D=I-P_D */ scalar_value = 1.0; ierr = VecAXPY(pcis->vec1_B,scalar_value,pcis->vec2_B);CHKERRQ(ierr); scalar_value = -1.0; ierr = VecAXPY(pcis->vec1_B,scalar_value,test_vec);CHKERRQ(ierr); ierr = VecNorm(pcis->vec1_B,NORM_INFINITY,&real_value);CHKERRQ(ierr); ierr = VecDestroy(&test_vec);CHKERRQ(ierr); ierr = PetscViewerASCIISynchronizedPrintf(viewer,"D[%04d] CHECK infty norm of E_D + P_D - I: % 1.14e\n",rank,real_value);CHKERRQ(ierr); ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); /******************************************************************/ /* TEST E: It should holds R_D^TP_Dw=0 w\in\widetilde{W} */ /* eq.48 Mandel Tezaur and Dohrmann 2005 */ /******************************************************************/ ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr); ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr); scalar_value = 0.0; /* set zero at vertices */ for (i=0;i<n_vertices;i++) { array[vertex_indices[i]]=scalar_value; } ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr); /* Jump operator P_D : results stored in pcis->vec1_B */ ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); /* Action of B_delta */ ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr); ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr); ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); /* Action of B_Ddelta^T */ ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr); /* scaling */ ierr = PCBDDCScalingExtension(fetidpmat_ctx->pc,pcis->vec1_B,pcis->vec1_global);CHKERRQ(ierr); ierr = VecNorm(pcis->vec1_global,NORM_INFINITY,&real_value);CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(viewer,"E[coll]: CHECK infty norm of R^T_D P_D: % 1.14e\n",real_value);CHKERRQ(ierr); ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); if (!fully_redundant) { /******************************************************************/ /* TEST F: It should holds B_{delta}B^T_{D,delta}=I */ /* Corollary thm 14 Mandel Tezaur and Dohrmann 2005 */ /******************************************************************/ ierr = VecDuplicate(lambda_global,&test_vec);CHKERRQ(ierr); ierr = VecSetRandom(lambda_global,NULL);CHKERRQ(ierr); /* Action of B_Ddelta^T */ ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr); ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr); /* Action of B_delta */ ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr); ierr = VecSet(test_vec,0.0);CHKERRQ(ierr); ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,test_vec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,test_vec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr); scalar_value = -1.0; ierr = VecAXPY(lambda_global,scalar_value,test_vec);CHKERRQ(ierr); ierr = VecNorm(lambda_global,NORM_INFINITY,&real_value);CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(viewer,"E[coll]: CHECK infty norm of P^T_D - I: % 1.14e\n",real_value);CHKERRQ(ierr); ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); ierr = PetscViewerFlush(viewer);CHKERRQ(ierr); ierr = VecDestroy(&test_vec);CHKERRQ(ierr); } } /* final cleanup */ ierr = VecDestroy(&lambda_global);CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **argv) { TS ts; SNES snes_alg; PetscErrorCode ierr; PetscMPIInt size; Userctx user; PetscViewer Xview,Ybusview; Vec X; Mat J; PetscInt i; ierr = PetscInitialize(&argc,&argv,"petscoptions",help);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); if (size > 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Only for sequential runs"); user.neqs_gen = 9*ngen; /* # eqs. for generator subsystem */ user.neqs_net = 2*nbus; /* # eqs. for network subsystem */ user.neqs_pgrid = user.neqs_gen + user.neqs_net; /* Create indices for differential and algebraic equations */ PetscInt *idx2; ierr = PetscMalloc1(7*ngen,&idx2);CHKERRQ(ierr); for (i=0; i<ngen; i++) { idx2[7*i] = 9*i; idx2[7*i+1] = 9*i+1; idx2[7*i+2] = 9*i+2; idx2[7*i+3] = 9*i+3; idx2[7*i+4] = 9*i+6; idx2[7*i+5] = 9*i+7; idx2[7*i+6] = 9*i+8; } ierr = ISCreateGeneral(PETSC_COMM_WORLD,7*ngen,idx2,PETSC_COPY_VALUES,&user.is_diff);CHKERRQ(ierr); ierr = ISComplement(user.is_diff,0,user.neqs_pgrid,&user.is_alg);CHKERRQ(ierr); ierr = PetscFree(idx2);CHKERRQ(ierr); /* Read initial voltage vector and Ybus */ ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"X.bin",FILE_MODE_READ,&Xview);CHKERRQ(ierr); ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"Ybus.bin",FILE_MODE_READ,&Ybusview);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_WORLD,&user.V0);CHKERRQ(ierr); ierr = VecSetSizes(user.V0,PETSC_DECIDE,user.neqs_net);CHKERRQ(ierr); ierr = VecLoad(user.V0,Xview);CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&user.Ybus);CHKERRQ(ierr); ierr = MatSetSizes(user.Ybus,PETSC_DECIDE,PETSC_DECIDE,user.neqs_net,user.neqs_net);CHKERRQ(ierr); ierr = MatSetType(user.Ybus,MATBAIJ);CHKERRQ(ierr); /* ierr = MatSetBlockSize(user.Ybus,2);CHKERRQ(ierr); */ ierr = MatLoad(user.Ybus,Ybusview);CHKERRQ(ierr); /* Set run time options */ ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"Transient stability fault options","");CHKERRQ(ierr); { user.tfaulton = 1.0; user.tfaultoff = 1.2; user.Rfault = 0.0001; user.faultbus = 8; ierr = PetscOptionsReal("-tfaulton","","",user.tfaulton,&user.tfaulton,NULL);CHKERRQ(ierr); ierr = PetscOptionsReal("-tfaultoff","","",user.tfaultoff,&user.tfaultoff,NULL);CHKERRQ(ierr); ierr = PetscOptionsInt("-faultbus","","",user.faultbus,&user.faultbus,NULL);CHKERRQ(ierr); user.t0 = 0.0; user.tmax = 5.0; ierr = PetscOptionsReal("-t0","","",user.t0,&user.t0,NULL);CHKERRQ(ierr); ierr = PetscOptionsReal("-tmax","","",user.tmax,&user.tmax,NULL);CHKERRQ(ierr); } ierr = PetscOptionsEnd();CHKERRQ(ierr); ierr = PetscViewerDestroy(&Xview);CHKERRQ(ierr); ierr = PetscViewerDestroy(&Ybusview);CHKERRQ(ierr); /* Create DMs for generator and network subsystems */ ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,user.neqs_gen,1,1,NULL,&user.dmgen);CHKERRQ(ierr); ierr = DMSetOptionsPrefix(user.dmgen,"dmgen_");CHKERRQ(ierr); ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,user.neqs_net,1,1,NULL,&user.dmnet);CHKERRQ(ierr); ierr = DMSetOptionsPrefix(user.dmnet,"dmnet_");CHKERRQ(ierr); /* Create a composite DM packer and add the two DMs */ ierr = DMCompositeCreate(PETSC_COMM_WORLD,&user.dmpgrid);CHKERRQ(ierr); ierr = DMSetOptionsPrefix(user.dmpgrid,"pgrid_");CHKERRQ(ierr); ierr = DMCompositeAddDM(user.dmpgrid,user.dmgen);CHKERRQ(ierr); ierr = DMCompositeAddDM(user.dmpgrid,user.dmnet);CHKERRQ(ierr); ierr = DMCreateGlobalVector(user.dmpgrid,&X);CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&J);CHKERRQ(ierr); ierr = MatSetSizes(J,PETSC_DECIDE,PETSC_DECIDE,user.neqs_pgrid,user.neqs_pgrid);CHKERRQ(ierr); ierr = MatSetFromOptions(J);CHKERRQ(ierr); ierr = PreallocateJacobian(J,&user);CHKERRQ(ierr); /* Create matrix to save solutions at each time step */ user.stepnum = 0; ierr = MatCreateSeqDense(PETSC_COMM_SELF,user.neqs_pgrid+1,1002,NULL,&user.Sol);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Create timestepping solver context - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr); ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr); ierr = TSSetEquationType(ts,TS_EQ_DAE_IMPLICIT_INDEX1);CHKERRQ(ierr); ierr = TSARKIMEXSetFullyImplicit(ts,PETSC_TRUE);CHKERRQ(ierr); ierr = TSSetIFunction(ts,NULL,(TSIFunction) IFunction,&user);CHKERRQ(ierr); ierr = TSSetIJacobian(ts,J,J,(TSIJacobian)IJacobian,&user);CHKERRQ(ierr); ierr = TSSetApplicationContext(ts,&user);CHKERRQ(ierr); /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Set initial conditions - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ ierr = SetInitialGuess(X,&user);CHKERRQ(ierr); /* Just to set up the Jacobian structure */ Vec Xdot; MatStructure flg; ierr = VecDuplicate(X,&Xdot);CHKERRQ(ierr); ierr = IJacobian(ts,0.0,X,Xdot,0.0,J,J,&flg,&user);CHKERRQ(ierr); ierr = VecDestroy(&Xdot);CHKERRQ(ierr); /* Save initial solution */ PetscScalar *x,*mat; PetscInt idx=user.stepnum*(user.neqs_pgrid+1); ierr = MatDenseGetArray(user.Sol,&mat);CHKERRQ(ierr); ierr = VecGetArray(X,&x);CHKERRQ(ierr); mat[idx] = 0.0; ierr = PetscMemcpy(mat+idx+1,x,user.neqs_pgrid*sizeof(PetscScalar));CHKERRQ(ierr); ierr = MatDenseRestoreArray(user.Sol,&mat);CHKERRQ(ierr); ierr = VecRestoreArray(X,&x);CHKERRQ(ierr); user.stepnum++; ierr = TSSetDuration(ts,1000,user.tfaulton);CHKERRQ(ierr); ierr = TSSetInitialTimeStep(ts,0.0,0.01);CHKERRQ(ierr); ierr = TSSetFromOptions(ts);CHKERRQ(ierr); ierr = TSSetPostStep(ts,SaveSolution);CHKERRQ(ierr); user.alg_flg = PETSC_FALSE; /* Prefault period */ ierr = TSSolve(ts,X);CHKERRQ(ierr); /* Create the nonlinear solver for solving the algebraic system */ /* Note that although the algebraic system needs to be solved only for Idq and V, we reuse the entire system including xgen. The xgen variables are held constant by setting their residuals to 0 and putting a 1 on the Jacobian diagonal for xgen rows */ Vec F_alg; ierr = VecDuplicate(X,&F_alg);CHKERRQ(ierr); ierr = SNESCreate(PETSC_COMM_WORLD,&snes_alg);CHKERRQ(ierr); ierr = SNESSetFunction(snes_alg,F_alg,AlgFunction,&user);CHKERRQ(ierr); ierr = MatZeroEntries(J);CHKERRQ(ierr); ierr = SNESSetJacobian(snes_alg,J,J,AlgJacobian,&user);CHKERRQ(ierr); ierr = SNESSetOptionsPrefix(snes_alg,"alg_");CHKERRQ(ierr); ierr = SNESSetFromOptions(snes_alg);CHKERRQ(ierr); /* Apply disturbance - resistive fault at user.faultbus */ /* This is done by adding shunt conductance to the diagonal location in the Ybus matrix */ PetscInt row_loc,col_loc; PetscScalar val; row_loc = 2*user.faultbus; col_loc = 2*user.faultbus+1; /* Location for G */ val = 1/user.Rfault; ierr = MatSetValues(user.Ybus,1,&row_loc,1,&col_loc,&val,ADD_VALUES);CHKERRQ(ierr); row_loc = 2*user.faultbus+1; col_loc = 2*user.faultbus; /* Location for G */ val = 1/user.Rfault; ierr = MatSetValues(user.Ybus,1,&row_loc,1,&col_loc,&val,ADD_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(user.Ybus,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(user.Ybus,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); user.alg_flg = PETSC_TRUE; /* Solve the algebraic equations */ ierr = SNESSolve(snes_alg,NULL,X);CHKERRQ(ierr); /* Save fault-on solution */ idx = user.stepnum*(user.neqs_pgrid+1); ierr = MatDenseGetArray(user.Sol,&mat);CHKERRQ(ierr); ierr = VecGetArray(X,&x);CHKERRQ(ierr); mat[idx] = user.tfaulton; ierr = PetscMemcpy(mat+idx+1,x,user.neqs_pgrid*sizeof(PetscScalar));CHKERRQ(ierr); ierr = MatDenseRestoreArray(user.Sol,&mat);CHKERRQ(ierr); ierr = VecRestoreArray(X,&x);CHKERRQ(ierr); user.stepnum++; /* Disturbance period */ ierr = TSSetDuration(ts,1000,user.tfaultoff);CHKERRQ(ierr); ierr = TSSetInitialTimeStep(ts,user.tfaulton,.01);CHKERRQ(ierr); user.alg_flg = PETSC_FALSE; ierr = TSSolve(ts,X);CHKERRQ(ierr); /* Remove the fault */ row_loc = 2*user.faultbus; col_loc = 2*user.faultbus+1; val = -1/user.Rfault; ierr = MatSetValues(user.Ybus,1,&row_loc,1,&col_loc,&val,ADD_VALUES);CHKERRQ(ierr); row_loc = 2*user.faultbus+1; col_loc = 2*user.faultbus; val = -1/user.Rfault; ierr = MatSetValues(user.Ybus,1,&row_loc,1,&col_loc,&val,ADD_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(user.Ybus,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(user.Ybus,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatZeroEntries(J);CHKERRQ(ierr); user.alg_flg = PETSC_TRUE; /* Solve the algebraic equations */ ierr = SNESSolve(snes_alg,NULL,X);CHKERRQ(ierr); /* Save tfault off solution */ idx = user.stepnum*(user.neqs_pgrid+1); ierr = MatDenseGetArray(user.Sol,&mat);CHKERRQ(ierr); ierr = VecGetArray(X,&x);CHKERRQ(ierr); mat[idx] = user.tfaultoff; ierr = PetscMemcpy(mat+idx+1,x,user.neqs_pgrid*sizeof(PetscScalar));CHKERRQ(ierr); ierr = MatDenseRestoreArray(user.Sol,&mat);CHKERRQ(ierr); ierr = VecRestoreArray(X,&x);CHKERRQ(ierr); user.stepnum++; /* Post-disturbance period */ ierr = TSSetDuration(ts,1000,user.tmax);CHKERRQ(ierr); ierr = TSSetInitialTimeStep(ts,user.tfaultoff,.01);CHKERRQ(ierr); user.alg_flg = PETSC_TRUE; ierr = TSSolve(ts,X);CHKERRQ(ierr); ierr = MatAssemblyBegin(user.Sol,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(user.Sol,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); Mat A; PetscScalar *amat; ierr = MatCreateSeqDense(PETSC_COMM_SELF,user.neqs_pgrid+1,user.stepnum,NULL,&A);CHKERRQ(ierr); ierr = MatDenseGetArray(user.Sol,&mat);CHKERRQ(ierr); ierr = MatDenseGetArray(A,&amat);CHKERRQ(ierr); ierr = PetscMemcpy(amat,mat,(user.stepnum*(user.neqs_pgrid+1))*sizeof(PetscScalar));CHKERRQ(ierr); ierr = MatDenseRestoreArray(A,&amat);CHKERRQ(ierr); ierr = MatDenseRestoreArray(user.Sol,&mat);CHKERRQ(ierr); PetscViewer viewer; ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"out.bin",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr); ierr = MatView(A,viewer);CHKERRQ(ierr); ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = SNESDestroy(&snes_alg);CHKERRQ(ierr); ierr = VecDestroy(&F_alg);CHKERRQ(ierr); ierr = MatDestroy(&J);CHKERRQ(ierr); ierr = MatDestroy(&user.Ybus);CHKERRQ(ierr); ierr = MatDestroy(&user.Sol);CHKERRQ(ierr); ierr = VecDestroy(&X);CHKERRQ(ierr); ierr = VecDestroy(&user.V0);CHKERRQ(ierr); ierr = DMDestroy(&user.dmgen);CHKERRQ(ierr); ierr = DMDestroy(&user.dmnet);CHKERRQ(ierr); ierr = DMDestroy(&user.dmpgrid);CHKERRQ(ierr); ierr = ISDestroy(&user.is_diff);CHKERRQ(ierr); ierr = ISDestroy(&user.is_alg);CHKERRQ(ierr); ierr = TSDestroy(&ts);CHKERRQ(ierr); ierr = PetscFinalize(); return(0); }
/* Mm_ratio - ration of grid lines between fine and coarse grids. */ int main(int argc,char **argv) { PetscErrorCode ierr; AppCtx user; PetscInt Npx=PETSC_DECIDE,Npy=PETSC_DECIDE,Npz=PETSC_DECIDE; PetscMPIInt size,rank; PetscInt m,n,M,N,i,nrows; PetscScalar one = 1.0; PetscReal fill=2.0; Mat A,A_tmp,P,C,C1,C2; PetscScalar *array,none = -1.0,alpha; Vec x,v1,v2,v3,v4; PetscReal norm,norm_tmp,norm_tmp1,tol=100.*PETSC_MACHINE_EPSILON; PetscRandom rdm; PetscBool Test_MatMatMult=PETSC_TRUE,Test_MatPtAP=PETSC_TRUE,Test_3D=PETSC_TRUE,flg; const PetscInt *ia,*ja; ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr; ierr = PetscOptionsGetReal(NULL,NULL,"-tol",&tol,NULL);CHKERRQ(ierr); user.ratio = 2; user.coarse.mx = 20; user.coarse.my = 20; user.coarse.mz = 20; ierr = PetscOptionsGetInt(NULL,NULL,"-Mx",&user.coarse.mx,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-My",&user.coarse.my,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-Mz",&user.coarse.mz,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-ratio",&user.ratio,NULL);CHKERRQ(ierr); if (user.coarse.mz) Test_3D = PETSC_TRUE; user.fine.mx = user.ratio*(user.coarse.mx-1)+1; user.fine.my = user.ratio*(user.coarse.my-1)+1; user.fine.mz = user.ratio*(user.coarse.mz-1)+1; ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-Npx",&Npx,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-Npy",&Npy,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,NULL,"-Npz",&Npz,NULL);CHKERRQ(ierr); /* Set up distributed array for fine grid */ if (!Test_3D) { ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,Npx,Npy,1,1,NULL,NULL,&user.fine.da);CHKERRQ(ierr); } else { ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,user.fine.mz,Npx,Npy,Npz,1,1,NULL,NULL,NULL,&user.fine.da);CHKERRQ(ierr); } ierr = DMSetFromOptions(user.fine.da);CHKERRQ(ierr); ierr = DMSetUp(user.fine.da);CHKERRQ(ierr); /* Test DMCreateMatrix() */ /*------------------------------------------------------------*/ ierr = DMSetMatType(user.fine.da,MATAIJ);CHKERRQ(ierr); ierr = DMCreateMatrix(user.fine.da,&A);CHKERRQ(ierr); ierr = DMSetMatType(user.fine.da,MATBAIJ);CHKERRQ(ierr); ierr = DMCreateMatrix(user.fine.da,&C);CHKERRQ(ierr); ierr = MatConvert(C,MATAIJ,MAT_INITIAL_MATRIX,&A_tmp);CHKERRQ(ierr); /* not work for mpisbaij matrix! */ ierr = MatEqual(A,A_tmp,&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"A != C"); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = MatDestroy(&A_tmp);CHKERRQ(ierr); /*------------------------------------------------------------*/ ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr); ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr); /* if (!rank) printf("A %d, %d\n",M,N); */ /* set val=one to A */ if (size == 1) { ierr = MatGetRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr); if (flg) { ierr = MatSeqAIJGetArray(A,&array);CHKERRQ(ierr); for (i=0; i<ia[nrows]; i++) array[i] = one; ierr = MatSeqAIJRestoreArray(A,&array);CHKERRQ(ierr); } ierr = MatRestoreRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr); } else { Mat AA,AB; ierr = MatMPIAIJGetSeqAIJ(A,&AA,&AB,NULL);CHKERRQ(ierr); ierr = MatGetRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr); if (flg) { ierr = MatSeqAIJGetArray(AA,&array);CHKERRQ(ierr); for (i=0; i<ia[nrows]; i++) array[i] = one; ierr = MatSeqAIJRestoreArray(AA,&array);CHKERRQ(ierr); } ierr = MatRestoreRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr); ierr = MatGetRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr); if (flg) { ierr = MatSeqAIJGetArray(AB,&array);CHKERRQ(ierr); for (i=0; i<ia[nrows]; i++) array[i] = one; ierr = MatSeqAIJRestoreArray(AB,&array);CHKERRQ(ierr); } ierr = MatRestoreRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr); } /* ierr = MatView(A, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */ /* Set up distributed array for coarse grid */ if (!Test_3D) { ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,Npx,Npy,1,1,NULL,NULL,&user.coarse.da);CHKERRQ(ierr); } else { ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,user.coarse.mz,Npx,Npy,Npz, 1,1,NULL,NULL,NULL,&user.coarse.da);CHKERRQ(ierr); } ierr = DMSetFromOptions(user.coarse.da);CHKERRQ(ierr); ierr = DMSetUp(user.coarse.da);CHKERRQ(ierr); /* Create interpolation between the fine and coarse grids */ ierr = DMCreateInterpolation(user.coarse.da,user.fine.da,&P,NULL);CHKERRQ(ierr); ierr = MatGetLocalSize(P,&m,&n);CHKERRQ(ierr); ierr = MatGetSize(P,&M,&N);CHKERRQ(ierr); /* if (!rank) printf("P %d, %d\n",M,N); */ /* Create vectors v1 and v2 that are compatible with A */ ierr = VecCreate(PETSC_COMM_WORLD,&v1);CHKERRQ(ierr); ierr = MatGetLocalSize(A,&m,NULL);CHKERRQ(ierr); ierr = VecSetSizes(v1,m,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(v1);CHKERRQ(ierr); ierr = VecDuplicate(v1,&v2);CHKERRQ(ierr); ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr); ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr); /* Test MatMatMult(): C = A*P */ /*----------------------------*/ if (Test_MatMatMult) { ierr = MatDuplicate(A,MAT_COPY_VALUES,&A_tmp);CHKERRQ(ierr); ierr = MatMatMult(A_tmp,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr); /* Test MAT_REUSE_MATRIX - reuse symbolic C */ alpha=1.0; for (i=0; i<2; i++) { alpha -= 0.1; ierr = MatScale(A_tmp,alpha);CHKERRQ(ierr); ierr = MatMatMult(A_tmp,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr); } /* Free intermediate data structures created for reuse of C=Pt*A*P */ ierr = MatFreeIntermediateDataStructures(C);CHKERRQ(ierr); /* Test MatDuplicate() */ /*----------------------------*/ ierr = MatDuplicate(C,MAT_COPY_VALUES,&C1);CHKERRQ(ierr); ierr = MatDuplicate(C1,MAT_COPY_VALUES,&C2);CHKERRQ(ierr); ierr = MatDestroy(&C1);CHKERRQ(ierr); ierr = MatDestroy(&C2);CHKERRQ(ierr); /* Create vector x that is compatible with P */ ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr); ierr = MatGetLocalSize(P,NULL,&n);CHKERRQ(ierr); ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(x);CHKERRQ(ierr); norm = 0.0; for (i=0; i<10; i++) { ierr = VecSetRandom(x,rdm);CHKERRQ(ierr); ierr = MatMult(P,x,v1);CHKERRQ(ierr); ierr = MatMult(A_tmp,v1,v2);CHKERRQ(ierr); /* v2 = A*P*x */ ierr = MatMult(C,x,v1);CHKERRQ(ierr); /* v1 = C*x */ ierr = VecAXPY(v1,none,v2);CHKERRQ(ierr); ierr = VecNorm(v1,NORM_1,&norm_tmp);CHKERRQ(ierr); ierr = VecNorm(v2,NORM_1,&norm_tmp1);CHKERRQ(ierr); norm_tmp /= norm_tmp1; if (norm_tmp > norm) norm = norm_tmp; } if (norm >= tol && !rank) { ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMatMult(), |v1 - v2|/|v2|: %g\n",(double)norm);CHKERRQ(ierr); } ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = MatDestroy(&A_tmp);CHKERRQ(ierr); } /* Test P^T * A * P - MatPtAP() */ /*------------------------------*/ if (Test_MatPtAP) { ierr = MatPtAP(A,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr); ierr = MatGetLocalSize(C,&m,&n);CHKERRQ(ierr); /* Test MAT_REUSE_MATRIX - reuse symbolic C */ alpha=1.0; for (i=0; i<1; i++) { alpha -= 0.1; ierr = MatScale(A,alpha);CHKERRQ(ierr); ierr = MatPtAP(A,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr); } /* Free intermediate data structures created for reuse of C=Pt*A*P */ ierr = MatFreeIntermediateDataStructures(C);CHKERRQ(ierr); /* Test MatDuplicate() */ /*----------------------------*/ ierr = MatDuplicate(C,MAT_COPY_VALUES,&C1);CHKERRQ(ierr); ierr = MatDuplicate(C1,MAT_COPY_VALUES,&C2);CHKERRQ(ierr); ierr = MatDestroy(&C1);CHKERRQ(ierr); ierr = MatDestroy(&C2);CHKERRQ(ierr); /* Create vector x that is compatible with P */ ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr); ierr = MatGetLocalSize(P,&m,&n);CHKERRQ(ierr); ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(x);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_WORLD,&v3);CHKERRQ(ierr); ierr = VecSetSizes(v3,n,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(v3);CHKERRQ(ierr); ierr = VecDuplicate(v3,&v4);CHKERRQ(ierr); norm = 0.0; for (i=0; i<10; i++) { ierr = VecSetRandom(x,rdm);CHKERRQ(ierr); ierr = MatMult(P,x,v1);CHKERRQ(ierr); ierr = MatMult(A,v1,v2);CHKERRQ(ierr); /* v2 = A*P*x */ ierr = MatMultTranspose(P,v2,v3);CHKERRQ(ierr); /* v3 = Pt*A*P*x */ ierr = MatMult(C,x,v4);CHKERRQ(ierr); /* v3 = C*x */ ierr = VecAXPY(v4,none,v3);CHKERRQ(ierr); ierr = VecNorm(v4,NORM_1,&norm_tmp);CHKERRQ(ierr); ierr = VecNorm(v3,NORM_1,&norm_tmp1);CHKERRQ(ierr); norm_tmp /= norm_tmp1; if (norm_tmp > norm) norm = norm_tmp; } if (norm >= tol && !rank) { ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatPtAP(), |v3 - v4|/|v3|: %g\n",(double)norm);CHKERRQ(ierr); } ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = VecDestroy(&v3);CHKERRQ(ierr); ierr = VecDestroy(&v4);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); } /* Clean up */ ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr); ierr = VecDestroy(&v1);CHKERRQ(ierr); ierr = VecDestroy(&v2);CHKERRQ(ierr); ierr = DMDestroy(&user.fine.da);CHKERRQ(ierr); ierr = DMDestroy(&user.coarse.da);CHKERRQ(ierr); ierr = MatDestroy(&P);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
PetscInt main(PetscInt argc,char **args) { ptrdiff_t N0=256,N1=256,N2=256,N3=2,dim[4]; fftw_plan bplan,fplan; fftw_complex *out; double *in1,*in2; ptrdiff_t alloc_local,local_n0,local_0_start; ptrdiff_t local_n1,local_1_start; PetscInt i,j,indx,n1; PetscInt size,rank,n,N,*in,N_factor,NM; PetscScalar *data_fin,value1,one=1.57,zero=0.0; PetscScalar a,*x_arr,*y_arr,*z_arr,enorm; Vec fin,fout,fout1,ini,final; PetscRandom rnd; PetscErrorCode ierr; VecScatter vecscat,vecscat1; IS indx1,indx2; PetscInt *indx3,k,l,*indx4; PetscInt low,tempindx,tempindx1; ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr); #if defined(PETSC_USE_COMPLEX) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This example requires real numbers. Your current scalar type is complex"); #endif ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr); PetscRandomCreate(PETSC_COMM_WORLD,&rnd); alloc_local = fftw_mpi_local_size_3d_transposed(N0,N1,N2/2+1,PETSC_COMM_WORLD,&local_n0,&local_0_start,&local_n1,&local_1_start); /* printf("The value alloc_local is %ld from process %d\n",alloc_local,rank); */ printf("The value local_n0 is %ld from process %d\n",local_n0,rank); /* printf("The value local_0_start is %ld from process %d\n",local_0_start,rank);*/ /* printf("The value local_n1 is %ld from process %d\n",local_n1,rank); */ /* printf("The value local_1_start is %ld from process %d\n",local_1_start,rank);*/ /* Allocate space for input and output arrays */ in1=(double*)fftw_malloc(sizeof(double)*alloc_local*2); in2=(double*)fftw_malloc(sizeof(double)*alloc_local*2); out=(fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local); N=2*N0*N1*(N2/2+1);N_factor=N0*N1*N2; n=2*local_n0*N1*(N2/2+1);n1=local_n1*N0*2*N1; /* printf("The value N is %d from process %d\n",N,rank); */ /* printf("The value n is %d from process %d\n",n,rank); */ /* printf("The value n1 is %d from process %d\n",n1,rank); */ /* Creating data vector and accompanying array with VeccreateMPIWithArray */ ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,n,N,(PetscScalar*)in1,&fin);CHKERRQ(ierr); ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,n,N,(PetscScalar*)out,&fout);CHKERRQ(ierr); ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,n,N,(PetscScalar*)in2,&fout1);CHKERRQ(ierr); /* VecGetSize(fin,&size); */ /* printf("The size is %d\n",size); */ VecSet(fin,one); VecSet(fout,zero); VecSet(fout1,zero); VecAssemblyBegin(fin); VecAssemblyEnd(fin); /* VecView(fin,PETSC_VIEWER_STDOUT_WORLD); */ VecGetArray(fin,&x_arr); VecGetArray(fout1,&z_arr); VecGetArray(fout,&y_arr); fplan=fftw_mpi_plan_dft_r2c_3d(N0,N1,N2,(double*)x_arr,(fftw_complex*)y_arr,PETSC_COMM_WORLD,FFTW_ESTIMATE); bplan=fftw_mpi_plan_dft_c2r_3d(N0,N1,N2,(fftw_complex*)y_arr,(double*)z_arr,PETSC_COMM_WORLD,FFTW_ESTIMATE); fftw_execute(fplan); fftw_execute(bplan); VecRestoreArray(fin,&x_arr); VecRestoreArray(fout1,&z_arr); VecRestoreArray(fout,&y_arr); /* a = 1.0/(PetscReal)N_factor; */ /* ierr = VecScale(fout1,a);CHKERRQ(ierr); */ VecCreate(PETSC_COMM_WORLD,&ini); VecCreate(PETSC_COMM_WORLD,&final); VecSetSizes(ini,local_n0*N1*N2,N_factor); VecSetSizes(final,local_n0*N1*N2,N_factor); /* VecSetSizes(ini,PETSC_DECIDE,N_factor); */ /* VecSetSizes(final,PETSC_DECIDE,N_factor); */ VecSetFromOptions(ini); VecSetFromOptions(final); if (N2%2==0) NM=N2+2; else NM=N2+1; ierr = VecGetOwnershipRange(fin,&low,NULL); printf("The local index is %d from %d\n",low,rank); ierr = PetscMalloc(sizeof(PetscInt)*local_n0*N1*N2,&indx3); ierr = PetscMalloc(sizeof(PetscInt)*local_n0*N1*N2,&indx4); for (i=0; i<local_n0; i++) { for (j=0;j<N1;j++) { for (k=0;k<N2;k++) { tempindx = i*N1*N2 + j*N2 + k; tempindx1 = i*N1*NM + j*NM + k; indx3[tempindx]=local_0_start*N1*N2+tempindx; indx4[tempindx]=low+tempindx1; } /* printf("index3 %d from proc %d is \n",indx3[tempindx],rank); */ /* printf("index4 %d from proc %d is \n",indx4[tempindx],rank); */ } } VecGetValues(fin,local_n0*N1*N2,indx4,x_arr); VecSetValues(ini,local_n0*N1*N2,indx3,x_arr,INSERT_VALUES); VecAssemblyBegin(ini); VecAssemblyEnd(ini); VecGetValues(fout1,local_n0*N1*N2,indx4,y_arr); VecSetValues(final,local_n0*N1*N2,indx3,y_arr,INSERT_VALUES); VecAssemblyBegin(final); VecAssemblyEnd(final); printf("The local index value is %ld from %d",local_n0*N1*N2,rank); /* for (i=0;i<N0;i++) { for (j=0;j<N1;j++) { indx=i*N1*NM+j*NM; ISCreateStride(PETSC_COMM_WORLD,N2,indx,1,&indx1); indx=i*N1*N2+j*N2; ISCreateStride(PETSC_COMM_WORLD,N2,indx,1,&indx2); VecScatterCreate(fin,indx1,ini,indx2,&vecscat); VecScatterBegin(vecscat,fin,ini,INSERT_VALUES,SCATTER_FORWARD); VecScatterEnd(vecscat,fin,ini,INSERT_VALUES,SCATTER_FORWARD); VecScatterCreate(fout1,indx1,final,indx2,&vecscat1); VecScatterBegin(vecscat1,fout1,final,INSERT_VALUES,SCATTER_FORWARD); VecScatterEnd(vecscat1,fout1,final,INSERT_VALUES,SCATTER_FORWARD); } } */ a = 1.0/(PetscReal)N_factor; ierr = VecScale(fout1,a);CHKERRQ(ierr); ierr = VecScale(final,a);CHKERRQ(ierr); VecAssemblyBegin(ini); VecAssemblyEnd(ini); VecAssemblyBegin(final); VecAssemblyEnd(final); /* VecView(final,PETSC_VIEWER_STDOUT_WORLD); */ ierr = VecAXPY(final,-1.0,ini);CHKERRQ(ierr); ierr = VecNorm(final,NORM_1,&enorm);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD," Error norm of |x - z| = %e\n",enorm);CHKERRQ(ierr); fftw_destroy_plan(fplan); fftw_destroy_plan(bplan); fftw_free(in1); ierr = VecDestroy(&fin);CHKERRQ(ierr); fftw_free(out); ierr = VecDestroy(&fout);CHKERRQ(ierr); fftw_free(in2); ierr = VecDestroy(&fout1);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }