PetscVector<Number> assemble_coupled_rhs (EquationSystems& es,Tree& tree, Mesh& mesh)
{
std::cout<< "Assembling big rhs " <<std::endl;
TransientLinearImplicitSystem & system =
es.add_system<TransientLinearImplicitSystem> ("Newton-update");
NumericVector< Number > &solution_in= *(system.solution);
NumericVector< Number > &rhs_in = *(system.rhs);
Number size_mat = system.rhs->size();
Real size_t=tree.number_nodes+tree.number_edges;
//Sort out rhs - put into big vector
Vec x;
VecCreate(PETSC_COMM_WORLD,&x);
PetscObjectSetName((PetscObject) x, "Solution");
VecSetSizes(x,PETSC_DECIDE,size_mat);
VecSetFromOptions(x);
PetscVector<Number> xp(x) ;
xp.add(1,solution_in);
Vec r;
VecCreate(PETSC_COMM_WORLD,&r);
PetscObjectSetName((PetscObject) r, "rhs");
VecSetSizes(r,PETSC_DECIDE,size_mat);
VecSetFromOptions(r);
PetscVector<Number> rp(r) ;
rp.add(1,rhs_in);
Vec rt;
rt=tree.make_tree_rhs( );
PetscVector<Number> rtp(rt) ;
Vec big_r;
VecCreate(PETSC_COMM_WORLD,&big_r);
PetscObjectSetName((PetscObject) big_r, "big_rhs");
VecSetSizes(big_r,PETSC_DECIDE,size_mat+size_t);
//VecSetSizes(big_r,PETSC_DECIDE,size_mat);
VecSetFromOptions(big_r);
// add system.rhs into big_rp
for (int i=0; i<size_mat; i++) {
//if(abs(rhs_in(i))>0){
VecSetValue(big_r,i,rp(i),INSERT_VALUES);
//}
}
// add extra tree rhs entries should all be 0 if p0=0
for (int i=size_mat; i<size_mat+size_t; i++) {
VecSetValue(big_r, i , rtp(i-size_mat) ,INSERT_VALUES);
}
PetscVector<Number> big_rp(big_r) ;
return big_rp;
}
int main(int argc, char** argv)
{
PC pc;
PetscErrorCode ierr;
PetscInt m, nn, M, j, k, ne = 4;
PetscReal* coords;
Vec x, rhs;
Mat A;
KSP ksp;
PetscMPIInt npe, rank;
PetscInitialize(&argc, &argv, NULL, NULL);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &npe);
CHKERRQ(ierr);
ierr = PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "Linear elasticity in 3D", "");
{
char nestring[256];
ierr = PetscSNPrintf(nestring, sizeof nestring, "number of elements in each direction, ne+1 must be a multiple of %D (sizes^{1/3})",
(PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5));
ierr = PetscOptionsInt("-ne", nestring, "", ne, &ne, NULL);
}
ierr = PetscOptionsEnd();
CHKERRQ(ierr);
const HpddmOption* const opt = HpddmOptionGet();
{
HpddmOptionParse(opt, argc, argv, rank == 0);
if (rank) HpddmOptionRemove(opt, "verbosity");
}
nn = ne + 1;
M = 3 * nn * nn * nn;
if (npe == 2) {
if (rank == 1)
m = 0;
else
m = nn * nn * nn;
npe = 1;
}
else {
m = nn * nn * nn / npe;
if (rank == npe - 1) m = nn * nn * nn - (npe - 1) * m;
}
m *= 3;
ierr = KSPCreate(PETSC_COMM_WORLD, &ksp);
CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);
CHKERRQ(ierr);
int i;
{
PetscInt Istart, Iend, jj, ic;
const PetscInt NP = (PetscInt)(PetscPowReal((PetscReal)npe, 1.0 / 3.0) + 0.5);
const PetscInt ipx = rank % NP, ipy = (rank % (NP * NP)) / NP, ipz = rank / (NP * NP);
const PetscInt Ni0 = ipx * (nn / NP), Nj0 = ipy * (nn / NP), Nk0 = ipz * (nn / NP);
const PetscInt Ni1 = Ni0 + (m > 0 ? (nn / NP) : 0), Nj1 = Nj0 + (nn / NP), Nk1 = Nk0 + (nn / NP);
PetscInt *d_nnz, *o_nnz, osz[4] = {0, 9, 15, 19}, nbc;
if (npe != NP * NP * NP) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "npe=%d: npe^{1/3} must be integer", npe);
if (nn != NP * (nn / NP)) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "-ne %d: (ne+1)%(npe^{1/3}) must equal zero", ne);
ierr = PetscMalloc1(m + 1, &d_nnz);
CHKERRQ(ierr);
ierr = PetscMalloc1(m + 1, &o_nnz);
CHKERRQ(ierr);
for (i = Ni0, ic = 0; i < Ni1; i++) {
for (j = Nj0; j < Nj1; j++) {
for (k = Nk0; k < Nk1; k++) {
nbc = 0;
if (i == Ni0 || i == Ni1 - 1) nbc++;
if (j == Nj0 || j == Nj1 - 1) nbc++;
if (k == Nk0 || k == Nk1 - 1) nbc++;
for (jj = 0; jj < 3; jj++, ic++) {
d_nnz[ic] = 3 * (27 - osz[nbc]);
o_nnz[ic] = 3 * osz[nbc];
}
}
}
}
if (ic != m) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "ic %D does not equal m %D", ic, m);
ierr = MatCreate(PETSC_COMM_WORLD, &A);
CHKERRQ(ierr);
ierr = MatSetSizes(A, m, m, M, M);
CHKERRQ(ierr);
ierr = MatSetBlockSize(A, 3);
CHKERRQ(ierr);
ierr = MatSetType(A, MATAIJ);
CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(A, 0, d_nnz);
CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(A, 0, d_nnz, 0, o_nnz);
CHKERRQ(ierr);
ierr = PetscFree(d_nnz);
CHKERRQ(ierr);
ierr = PetscFree(o_nnz);
CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A, &Istart, &Iend);
CHKERRQ(ierr);
if (m != Iend - Istart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "m %D does not equal Iend %D - Istart %D", m, Iend, Istart);
ierr = VecCreate(PETSC_COMM_WORLD, &x);
CHKERRQ(ierr);
ierr = VecSetSizes(x, m, M);
CHKERRQ(ierr);
ierr = VecSetBlockSize(x, 3);
CHKERRQ(ierr);
ierr = VecSetFromOptions(x);
CHKERRQ(ierr);
ierr = VecDuplicate(x, &rhs);
CHKERRQ(ierr);
ierr = PetscMalloc1(m + 1, &coords);
CHKERRQ(ierr);
coords[m] = -99.0;
PetscReal h = 1.0 / ne;
for (i = Ni0, ic = 0; i < Ni1; i++) {
for (j = Nj0; j < Nj1; j++) {
for (k = Nk0; k < Nk1; k++, ic++) {
coords[3 * ic] = h * (PetscReal)i;
coords[3 * ic + 1] = h * (PetscReal)j;
coords[3 * ic + 2] = h * (PetscReal)k;
}
}
}
}
PetscReal s_r[SIZE_ARRAY_R] = {30, 0.1, 20, 10};
PetscReal x_r[SIZE_ARRAY_R] = {0.5, 0.4, 0.4, 0.4};
PetscReal y_r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4};
PetscReal z_r[SIZE_ARRAY_R] = {0.5, 0.45, 0.4, 0.35};
PetscReal r[SIZE_ARRAY_R] = {0.5, 0.5, 0.4, 0.4};
AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
MatNullSpace matnull;
Vec vec_coords;
PetscScalar* c;
ierr = VecCreate(MPI_COMM_WORLD, &vec_coords);
CHKERRQ(ierr);
ierr = VecSetBlockSize(vec_coords, 3);
CHKERRQ(ierr);
ierr = VecSetSizes(vec_coords, m, PETSC_DECIDE);
CHKERRQ(ierr);
ierr = VecSetUp(vec_coords);
CHKERRQ(ierr);
ierr = VecGetArray(vec_coords, &c);
CHKERRQ(ierr);
for (i = 0; i < m; i++) c[i] = coords[i];
ierr = VecRestoreArray(vec_coords, &c);
CHKERRQ(ierr);
ierr = MatNullSpaceCreateRigidBody(vec_coords, &matnull);
CHKERRQ(ierr);
ierr = MatSetNearNullSpace(A, matnull);
CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&matnull);
CHKERRQ(ierr);
ierr = VecDestroy(&vec_coords);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
double time = MPI_Wtime();
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime() - time;
ierr = PetscPrintf(PETSC_COMM_WORLD, "--- PC setup = %f\n", time);
CHKERRQ(ierr);
float t_time[SIZE_ARRAY_R];
int t_its[SIZE_ARRAY_R];
{
{
ierr = KSPSolve(ksp, rhs, x);
CHKERRQ(ierr);
ierr = KSPReset(ksp);
CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = VecZeroEntries(x);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime();
ierr = KSPSolve(ksp, rhs, x);
CHKERRQ(ierr);
MPI_Barrier(PETSC_COMM_WORLD);
t_time[i] = MPI_Wtime() - time;
PetscInt its;
ierr = KSPGetIterationNumber(ksp, &its);
CHKERRQ(ierr);
t_its[i] = its;
ierr = ComputeError(A, rhs, x);
CHKERRQ(ierr);
if (i == (SIZE_ARRAY_R - 1))
AssembleSystem(A, rhs, s_r[0], x_r[0], y_r[0], z_r[0], r[0], ne, npe, rank, nn, m);
else
AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]);
CHKERRQ(ierr);
if (i > 0) {
t_its[0] += t_its[i];
t_time[0] += t_time[i];
}
}
if (SIZE_ARRAY_R > 1) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]);
CHKERRQ(ierr);
}
}
{
ierr = KSPGetPC(ksp, &pc);
CHKERRQ(ierr);
HpddmCustomOperator H;
H._A = A;
H._M = pc;
H._mv = mv;
H._precond = precond;
H._b = rhs;
H._x = x;
int n;
MatGetLocalSize(A, &n, NULL);
{
ierr = VecZeroEntries(x);
K* pt_rhs;
K* pt_x;
VecGetArray(rhs, &pt_rhs);
VecGetArray(x, &pt_x);
int previous = HpddmOptionVal(opt, "verbosity");
if (previous > 0) HpddmOptionRemove(opt, "verbosity");
HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD);
if (previous > 0) {
char buffer[20];
snprintf(buffer, 20, "%d", previous);
char* concat = malloc(strlen("-hpddm_verbosity ") + strlen(buffer) + 1);
strcpy(concat, "-hpddm_verbosity ");
strcat(concat, buffer);
HpddmOptionParseString(opt, concat);
free(concat);
}
VecRestoreArray(x, &pt_x);
VecRestoreArray(rhs, &pt_rhs);
previous = HpddmOptionVal(opt, "krylov_method");
if(previous == 4 || previous == 5) HpddmDestroyRecycling();
ierr = KSPReset(ksp);
CHKERRQ(ierr);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp, PETSC_TRUE);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = VecZeroEntries(x);
CHKERRQ(ierr);
K* pt_rhs;
K* pt_x;
VecGetArray(rhs, &pt_rhs);
VecGetArray(x, &pt_x);
MPI_Barrier(PETSC_COMM_WORLD);
time = MPI_Wtime();
t_its[i] = HpddmCustomOperatorSolve(&H, n, H._mv, H._precond, pt_rhs, pt_x, 1, &PETSC_COMM_WORLD);
MPI_Barrier(PETSC_COMM_WORLD);
t_time[i] = MPI_Wtime() - time;
VecRestoreArray(x, &pt_x);
VecRestoreArray(rhs, &pt_rhs);
ierr = ComputeError(A, rhs, x);
CHKERRQ(ierr);
if (i != (SIZE_ARRAY_R - 1)) {
AssembleSystem(A, rhs, s_r[i + 1], x_r[i + 1], y_r[i + 1], z_r[i + 1], r[i + 1], ne, npe, rank, nn, m);
ierr = KSPSetOperators(ksp, A, A);
CHKERRQ(ierr);
ierr = KSPSetUp(ksp);
CHKERRQ(ierr);
}
}
for (i = 0; i < SIZE_ARRAY_R; ++i) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "%d\t%d\t%f\n", i + 1, t_its[i], t_time[i]);
CHKERRQ(ierr);
if (i > 0) {
t_its[0] += t_its[i];
t_time[0] += t_time[i];
}
}
if (SIZE_ARRAY_R > 1) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "------------------------\n\t%d\t%f\n", t_its[0], t_time[0]);
CHKERRQ(ierr);
}
}
ierr = KSPDestroy(&ksp);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&rhs);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = PetscFree(coords);
CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
For MATSEQDENSE matrix, the factorization is just a thin wrapper to LAPACK \n\n";
#include <petscmat.h>
#undef __FUNCT__
#define __FUNCT__ "main"
int main(int argc,char **argv)
{
Mat mat,F,RHS,SOLU;
MatInfo info;
PetscErrorCode ierr;
PetscInt m = 10,n = 10,i,j,rstart,rend,nrhs=2;
PetscScalar value = 1.0;
Vec x,y,b,ytmp;
PetscReal norm,tol=1.e-15;
PetscMPIInt size;
PetscScalar *rhs_array,*solu_array;
PetscRandom rand;
PetscScalar *array,rval;
PetscInitialize(&argc,&argv,(char*) 0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"This is a uniprocessor example only!");
/* create single vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&y);CHKERRQ(ierr);
ierr = VecSetSizes(y,PETSC_DECIDE,m);CHKERRQ(ierr);
ierr = VecSetFromOptions(y);CHKERRQ(ierr);
ierr = VecDuplicate(y,&x);CHKERRQ(ierr);
ierr = VecDuplicate(y,&ytmp);CHKERRQ(ierr);
ierr = VecSet(x,value);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
/* create multiple vectors RHS and SOLU */
ierr = MatCreate(PETSC_COMM_WORLD,&RHS);CHKERRQ(ierr);
ierr = MatSetSizes(RHS,PETSC_DECIDE,PETSC_DECIDE,n,nrhs);CHKERRQ(ierr);
ierr = MatSetType(RHS,MATDENSE);CHKERRQ(ierr);
ierr = MatSetFromOptions(RHS);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(RHS,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatDenseGetArray(RHS,&array);CHKERRQ(ierr);
for (j=0; j<nrhs; j++) {
for (i=0; i<n; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
array[n*j+i] = rval;
}
}
ierr = MatDenseRestoreArray(RHS,&array);CHKERRQ(ierr);
ierr = MatDuplicate(RHS,MAT_DO_NOT_COPY_VALUES,&SOLU);CHKERRQ(ierr);
/* create matrix */
ierr = MatCreateSeqDense(PETSC_COMM_WORLD,m,n,NULL,&mat);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(mat,&rstart,&rend);CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
value = (PetscReal)i+1;
ierr = MatSetValues(mat,1,&i,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatGetInfo(mat,MAT_LOCAL,&info);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"matrix nonzeros = %D, allocated nonzeros = %D\n",
(PetscInt)info.nz_used,(PetscInt)info.nz_allocated);CHKERRQ(ierr);
/* Cholesky factorization - perm and factinfo are ignored by LAPACK */
/* in-place Cholesky */
ierr = MatMult(mat,x,b);CHKERRQ(ierr);
ierr = MatDuplicate(mat,MAT_COPY_VALUES,&F);CHKERRQ(ierr);
ierr = MatCholeskyFactor(F,0,0);CHKERRQ(ierr);
ierr = MatSolve(F,b,y);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for Cholesky %G\n",norm);CHKERRQ(ierr);
}
/* out-place Cholesky */
ierr = MatGetFactor(mat,MATSOLVERPETSC,MAT_FACTOR_CHOLESKY,&F);CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(F,mat,0,0);CHKERRQ(ierr);
ierr = MatCholeskyFactorNumeric(F,mat,0);CHKERRQ(ierr);
ierr = MatSolve(F,b,y);CHKERRQ(ierr);
value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for Cholesky %G\n",norm);CHKERRQ(ierr);
}
ierr = MatDestroy(&F);CHKERRQ(ierr);
/* LU factorization - perms and factinfo are ignored by LAPACK */
i = m-1; value = 1.0;
ierr = MatSetValues(mat,1,&i,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatMult(mat,x,b);CHKERRQ(ierr);
ierr = MatDuplicate(mat,MAT_COPY_VALUES,&F);CHKERRQ(ierr);
/* in-place LU */
ierr = MatLUFactor(F,0,0,0);CHKERRQ(ierr);
ierr = MatSolve(F,b,y);CHKERRQ(ierr);
value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for LU %G\n",norm);CHKERRQ(ierr);
}
ierr = MatMatSolve(F,RHS,SOLU);CHKERRQ(ierr);
ierr = MatDenseGetArray(SOLU,&solu_array);CHKERRQ(ierr);
ierr = MatDenseGetArray(RHS,&rhs_array);CHKERRQ(ierr);
for (j=0; j<nrhs; j++) {
ierr = VecPlaceArray(y,solu_array+j*m);CHKERRQ(ierr);
ierr = VecPlaceArray(b,rhs_array+j*m);CHKERRQ(ierr);
ierr = MatMult(mat,y,ytmp);CHKERRQ(ierr);
ierr = VecAXPY(ytmp,-1.0,b);CHKERRQ(ierr); /* ytmp = mat*SOLU[:,j] - RHS[:,j] */
ierr = VecNorm(ytmp,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: Norm of residual for LU %G\n",norm);CHKERRQ(ierr);
}
ierr = VecResetArray(b);CHKERRQ(ierr);
ierr = VecResetArray(y);CHKERRQ(ierr);
}
ierr = MatDenseRestoreArray(RHS,&rhs_array);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(SOLU,&solu_array);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
/* out-place LU */
ierr = MatGetFactor(mat,MATSOLVERPETSC,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(F,mat,0,0,0);CHKERRQ(ierr);
ierr = MatLUFactorNumeric(F,mat,0);CHKERRQ(ierr);
ierr = MatSolve(F,b,y);CHKERRQ(ierr);
value = -1.0; ierr = VecAXPY(y,value,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: Norm of error for LU %G\n",norm);CHKERRQ(ierr);
}
/* free space */
ierr = MatDestroy(&F);CHKERRQ(ierr);
ierr = MatDestroy(&mat);CHKERRQ(ierr);
ierr = MatDestroy(&RHS);CHKERRQ(ierr);
ierr = MatDestroy(&SOLU);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&ytmp);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Vec x, b, u; /* approx solution, RHS, exact solution */
Mat A,B,C; /* linear system matrix */
KSP ksp; /* linear solver context */
PC pc; /* preconditioner context */
PetscReal norm; /* norm of solution error */
PetscErrorCode ierr;
PetscInt i,n = 20,col[3],its;
PetscMPIInt size;
PetscScalar neg_one = -1.0,one = 1.0,value[3];
PetscBool nonzeroguess = PETSC_FALSE;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,1,"This is a uniprocessor example only!");
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-nonzero_guess",&nonzeroguess,NULL);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compute the matrix and right-hand-side vector that define
the linear system, Ax = b.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create vectors. Note that we form 1 vector from scratch and
then duplicate as needed.
*/
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) x, "Solution");CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);CHKERRQ(ierr);
/*
Create matrix. When using MatCreate(), the matrix format can
be specified at runtime.
Performance tuning note: For problems of substantial size,
preallocation of matrix memory is crucial for attaining good
performance. See the matrix chapter of the users manual for details.
*/
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/*
Assemble matrix
*/
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0] = n - 2; col[1] = n - 1;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0; col[0] = 0; col[1] = 1; value[0] = 2.0; value[1] = -1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
jroman: added matrices
*/
ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatSetUp(B);CHKERRQ(ierr);
for (i=0; i<n; i++) {
ierr = MatSetValue(B,i,i,value[1],INSERT_VALUES);CHKERRQ(ierr);
if (n-i+n/3<n) {
ierr = MatSetValue(B,n-i+n/3,i,value[0],INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValue(B,i,n-i+n/3,value[0],INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatDuplicate(A,MAT_COPY_VALUES,&C);CHKERRQ(ierr);
ierr = MatAXPY(C,2.0,B,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
/*
Set exact solution; then compute right-hand-side vector.
*/
ierr = VecSet(u,one);CHKERRQ(ierr);
ierr = MatMult(C,u,b);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the linear solver and set various options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp,C,C);CHKERRQ(ierr);
/*
Set linear solver defaults for this problem (optional).
- By extracting the KSP and PC contexts from the KSP context,
we can then directly call any KSP and PC routines to set
various options.
- The following four statements are optional; all of these
parameters could alternatively be specified at runtime via
KSPSetFromOptions();
*/
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCJACOBI);CHKERRQ(ierr);
ierr = KSPSetTolerances(ksp,1.e-5,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT);CHKERRQ(ierr);
/*
Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
These options will override those specified above as long as
KSPSetFromOptions() is called _after_ any other customization
routines.
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
if (nonzeroguess) {
PetscScalar p = .5;
ierr = VecSet(x,p);CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp,PETSC_TRUE);CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Solve linear system
*/
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Check solution and clean up
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Check the error
*/
ierr = VecAXPY(x,neg_one,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %g, Iterations %D\n",(double)norm,its);CHKERRQ(ierr);
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
/*
Always call PetscFinalize() before exiting a program. This routine
- finalizes the PETSc libraries as well as MPI
- provides summary and diagnostic information if certain runtime
options are chosen (e.g., -log_summary).
*/
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscMPIInt rank,size;
PetscInt nlocal = 6,nghost = 2,ifrom[2],i,rstart,rend;
PetscErrorCode ierr;
PetscBool flg,flg2;
PetscScalar value,*array,*tarray=0;
Vec lx,gx,gxs;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);
if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);
CHKERRQ(ierr);
if (size != 2) SETERRQ(PETSC_COMM_SELF,1,"Must run example with two processors\n");
/*
Construct a two dimensional graph connecting nlocal degrees of
freedom per processor. From this we will generate the global
indices of needed ghost values
For simplicity we generate the entire graph on each processor:
in real application the graph would stored in parallel, but this
example is only to demonstrate the management of ghost padding
with VecCreateGhost().
In this example we consider the vector as representing
degrees of freedom in a one dimensional grid with periodic
boundary conditions.
----Processor 1--------- ----Processor 2 --------
0 1 2 3 4 5 6 7 8 9 10 11
|----|
|-------------------------------------------------|
*/
if (!rank) {
ifrom[0] = 11;
ifrom[1] = 6;
} else {
ifrom[0] = 0;
ifrom[1] = 5;
}
/*
Create the vector with two slots for ghost points. Note that both
the local vector (lx) and the global vector (gx) share the same
array for storing vector values.
*/
ierr = PetscOptionsHasName(NULL,NULL,"-allocate",&flg);
CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,NULL,"-vecmpisetghost",&flg2);
CHKERRQ(ierr);
if (flg) {
ierr = PetscMalloc1(nlocal+nghost,&tarray);
CHKERRQ(ierr);
ierr = VecCreateGhostWithArray(PETSC_COMM_WORLD,nlocal,PETSC_DECIDE,nghost,ifrom,tarray,&gxs);
CHKERRQ(ierr);
} else if (flg2) {
ierr = VecCreate(PETSC_COMM_WORLD,&gxs);
CHKERRQ(ierr);
ierr = VecSetType(gxs,VECMPI);
CHKERRQ(ierr);
ierr = VecSetSizes(gxs,nlocal,PETSC_DECIDE);
CHKERRQ(ierr);
ierr = VecMPISetGhost(gxs,nghost,ifrom);
CHKERRQ(ierr);
} else {
ierr = VecCreateGhost(PETSC_COMM_WORLD,nlocal,PETSC_DECIDE,nghost,ifrom,&gxs);
CHKERRQ(ierr);
}
/*
Test VecDuplicate()
*/
ierr = VecDuplicate(gxs,&gx);
CHKERRQ(ierr);
ierr = VecDestroy(&gxs);
CHKERRQ(ierr);
/*
Access the local representation
*/
ierr = VecGhostGetLocalForm(gx,&lx);
CHKERRQ(ierr);
/*
Set the values from 0 to 12 into the "global" vector
*/
ierr = VecGetOwnershipRange(gx,&rstart,&rend);
CHKERRQ(ierr);
for (i=rstart; i<rend; i++) {
value = (PetscScalar) i;
ierr = VecSetValues(gx,1,&i,&value,INSERT_VALUES);
CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(gx);
CHKERRQ(ierr);
ierr = VecAssemblyEnd(gx);
CHKERRQ(ierr);
ierr = VecGhostUpdateBegin(gx,INSERT_VALUES,SCATTER_FORWARD);
CHKERRQ(ierr);
ierr = VecGhostUpdateEnd(gx,INSERT_VALUES,SCATTER_FORWARD);
CHKERRQ(ierr);
/*
Print out each vector, including the ghost padding region.
*/
ierr = VecGetArray(lx,&array);
CHKERRQ(ierr);
for (i=0; i<nlocal+nghost; i++) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"%D %g\n",i,(double)PetscRealPart(array[i]));
CHKERRQ(ierr);
}
ierr = VecRestoreArray(lx,&array);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
ierr = VecGhostRestoreLocalForm(gx,&lx);
CHKERRQ(ierr);
ierr = VecDestroy(&gx);
CHKERRQ(ierr);
if (flg) {
ierr = PetscFree(tarray);
CHKERRQ(ierr);
}
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat A;
Vec min,max,maxabs;
PetscInt m,n;
PetscInt imin[M],imax[M],imaxabs[M],indices[N],row;
PetscScalar values[N];
PetscErrorCode ierr;
MatType type;
PetscMPIInt size;
PetscBool doTest=PETSC_TRUE;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,M,N);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
row = 0;
indices[0] = 0; indices[1] = 1; indices[2] = 2; indices[3] = 3; indices[4] = 4; indices[5] = 5;
values[0] = -1.0; values[1] = 0.0; values[2] = 1.0; values[3] = 3.0; values[4] = 4.0; values[5] = -5.0;
ierr = MatSetValues(A,1,&row,6,indices,values,INSERT_VALUES);CHKERRQ(ierr);
row = 1;
ierr = MatSetValues(A,1,&row,3,indices,values,INSERT_VALUES);CHKERRQ(ierr);
row = 4;
ierr = MatSetValues(A,1,&row,1,indices+4,values+4,INSERT_VALUES);CHKERRQ(ierr);
row = 4;
ierr = MatSetValues(A,1,&row,2,indices+4,values+4,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatGetLocalSize(A, &m,&n);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&min);CHKERRQ(ierr);
ierr = VecSetSizes(min,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(min);CHKERRQ(ierr);
ierr = VecDuplicate(min,&max);CHKERRQ(ierr);
ierr = VecDuplicate(min,&maxabs);CHKERRQ(ierr);
/* Test MatGetRowMin, MatGetRowMax and MatGetRowMaxAbs */
if (size == 1) {
ierr = MatGetRowMin(A,min,imin);CHKERRQ(ierr);
ierr = MatGetRowMax(A,max,imax);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Minimums\n");CHKERRQ(ierr);
ierr = VecView(min,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximums\n");CHKERRQ(ierr);
ierr = VecView(max,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imax,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values\n");CHKERRQ(ierr);
ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imaxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
} else {
ierr = MatGetType(A,&type);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nMatrix type: %s\n",type);CHKERRQ(ierr);
/* AIJ */
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&doTest);CHKERRQ(ierr);
if (doTest) {
ierr = MatGetRowMaxAbs(A,maxabs,NULL);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values:\n");CHKERRQ(ierr);
ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* BAIJ */
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIBAIJ,&doTest);CHKERRQ(ierr);
if (doTest) {
ierr = MatGetRowMaxAbs(A,maxabs,NULL);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values:\n");CHKERRQ(ierr);
ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
}
if (size == 1) {
ierr = MatConvert(A,MATDENSE,MAT_INPLACE_MATRIX,&A);CHKERRQ(ierr);
ierr = MatGetRowMin(A,min,imin);CHKERRQ(ierr);
ierr = MatGetRowMax(A,max,imax);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Minimums\n");CHKERRQ(ierr);
ierr = VecView(min,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximums\n");CHKERRQ(ierr);
ierr = VecView(max,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imax,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values\n");CHKERRQ(ierr);
ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imaxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecDestroy(&min);CHKERRQ(ierr);
ierr = VecDestroy(&max);CHKERRQ(ierr);
ierr = VecDestroy(&maxabs);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
uses block index sets\n\n";
#include <petscvec.h>
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt bs=1,n=5,N,i,low;
PetscInt ix0[3] = {5,7,9},iy0[3] = {1,2,4},ix1[3] = {2,3,1},iy1[3] = {0,3,9};
PetscMPIInt size,rank;
PetscScalar *array;
Vec x,x1,y;
IS isx,isy;
VecScatter ctx;
VecScatterType type;
PetscBool flg;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (size <2) SETERRQ(PETSC_COMM_SELF,1,"Must run more than one processor");
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
n = bs*n;
/* Create vector x over shared memory */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetType(x,VECNODE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x,&low,NULL);CHKERRQ(ierr);
ierr = VecGetArray(x,&array);CHKERRQ(ierr);
for (i=0; i<n; i++) {
array[i] = (PetscScalar)(i + low);
}
ierr = VecRestoreArray(x,&array);CHKERRQ(ierr);
/* ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Test some vector functions */
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecGetSize(x,&N);CHKERRQ(ierr);
ierr = VecGetLocalSize(x,&n);CHKERRQ(ierr);
ierr = VecDuplicate(x,&x1);CHKERRQ(ierr);
ierr = VecCopy(x,x1);CHKERRQ(ierr);
ierr = VecEqual(x,x1,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PetscObjectComm((PetscObject)x),PETSC_ERR_ARG_WRONG,"x1 != x");
ierr = VecScale(x1,2.0);CHKERRQ(ierr);
ierr = VecSet(x1,10.0);CHKERRQ(ierr);
/* ierr = VecView(x1,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Create vector y over shared memory */
ierr = VecCreate(PETSC_COMM_WORLD,&y);CHKERRQ(ierr);
ierr = VecSetSizes(y,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetType(y,VECNODE);CHKERRQ(ierr);
ierr = VecSetFromOptions(y);CHKERRQ(ierr);
ierr = VecGetArray(y,&array);CHKERRQ(ierr);
for (i=0; i<n; i++) {
array[i] = -(PetscScalar) (i + 100*rank);
}
ierr = VecRestoreArray(y,&array);CHKERRQ(ierr);
ierr = VecAssemblyBegin(y);CHKERRQ(ierr);
ierr = VecAssemblyEnd(y);CHKERRQ(ierr);
/* ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Create two index sets */
if (!rank) {
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix0,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy0,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
} else {
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix1,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy1,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
}
if (rank == 10) {
ierr = PetscPrintf(PETSC_COMM_SELF,"\n[%d] isx:\n",rank);CHKERRQ(ierr);
ierr = ISView(isx,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"\n[%d] isy:\n",rank);CHKERRQ(ierr);
ierr = ISView(isy,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
/* Create Vector scatter */
ierr = VecScatterCreate(x,isx,y,isy,&ctx);CHKERRQ(ierr);
ierr = VecScatterSetFromOptions(ctx);CHKERRQ(ierr);
ierr = VecScatterGetType(ctx,&type);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"scatter type %s\n",type);CHKERRQ(ierr);
/* Test forward vecscatter */
ierr = VecSet(y,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nSCATTER_FORWARD y:\n");CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Test reverse vecscatter */
ierr = VecSet(x,0.0);CHKERRQ(ierr);
ierr = VecSet(y,0.0);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(y,&low,NULL);CHKERRQ(ierr);
ierr = VecGetArray(y,&array);CHKERRQ(ierr);
for (i=0; i<n; i++) {
array[i] = (PetscScalar)(i+ low);
}
ierr = VecRestoreArray(y,&array);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,y,x,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,y,x,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nSCATTER_REVERSE x:\n");CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Free objects */
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = ISDestroy(&isx);CHKERRQ(ierr);
ierr = ISDestroy(&isy);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&x1);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@
MatCreateSubMatrix - Creates a composite matrix that acts as a submatrix
Collective on Mat
Input Parameters:
+ A - matrix that we will extract a submatrix of
. isrow - rows to be present in the submatrix
- iscol - columns to be present in the submatrix
Output Parameters:
. newmat - new matrix
Level: developer
Notes:
Most will use MatGetSubMatrix which provides a more efficient representation if it is available.
.seealso: MatGetSubMatrix(), MatSubMatrixUpdate()
@*/
PetscErrorCode MatCreateSubMatrix(Mat A,IS isrow,IS iscol,Mat *newmat)
{
Vec left,right;
PetscInt m,n;
Mat N;
Mat_SubMatrix *Na;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(A,MAT_CLASSID,1);
PetscValidHeaderSpecific(isrow,IS_CLASSID,2);
PetscValidHeaderSpecific(iscol,IS_CLASSID,3);
PetscValidPointer(newmat,4);
*newmat = 0;
ierr = MatCreate(PetscObjectComm((PetscObject)A),&N);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrow,&m);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscol,&n);CHKERRQ(ierr);
ierr = MatSetSizes(N,m,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = PetscObjectChangeTypeName((PetscObject)N,MATSUBMATRIX);CHKERRQ(ierr);
ierr = PetscNewLog(N,&Na);CHKERRQ(ierr);
N->data = (void*)Na;
ierr = PetscObjectReference((PetscObject)A);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)isrow);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)iscol);CHKERRQ(ierr);
Na->A = A;
Na->isrow = isrow;
Na->iscol = iscol;
Na->scale = 1.0;
N->ops->destroy = MatDestroy_SubMatrix;
N->ops->mult = MatMult_SubMatrix;
N->ops->multadd = MatMultAdd_SubMatrix;
N->ops->multtranspose = MatMultTranspose_SubMatrix;
N->ops->multtransposeadd = MatMultTransposeAdd_SubMatrix;
N->ops->scale = MatScale_SubMatrix;
N->ops->diagonalscale = MatDiagonalScale_SubMatrix;
ierr = MatSetBlockSizesFromMats(N,A,A);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(N->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(N->cmap);CHKERRQ(ierr);
ierr = MatCreateVecs(A,&Na->rwork,&Na->lwork);CHKERRQ(ierr);
ierr = VecCreate(PetscObjectComm((PetscObject)isrow),&left);CHKERRQ(ierr);
ierr = VecCreate(PetscObjectComm((PetscObject)iscol),&right);CHKERRQ(ierr);
ierr = VecSetSizes(left,m,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetSizes(right,n,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetUp(left);CHKERRQ(ierr);
ierr = VecSetUp(right);CHKERRQ(ierr);
ierr = VecScatterCreate(Na->lwork,isrow,left,NULL,&Na->lrestrict);CHKERRQ(ierr);
ierr = VecScatterCreate(right,NULL,Na->rwork,iscol,&Na->rprolong);CHKERRQ(ierr);
ierr = VecDestroy(&left);CHKERRQ(ierr);
ierr = VecDestroy(&right);CHKERRQ(ierr);
N->assembled = PETSC_TRUE;
ierr = MatSetUp(N);CHKERRQ(ierr);
*newmat = N;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
Mat A,F,B,X,C,Aher,G;
Vec b,x,c,d,e;
PetscErrorCode ierr;
PetscInt m = 5,n,p,i,j,nrows,ncols;
PetscScalar *v,*barray,rval;
PetscReal norm,tol=1.e-12;
PetscMPIInt size,rank;
PetscRandom rand;
const PetscInt *rows,*cols;
IS isrows,iscols;
PetscBool mats_view=PETSC_FALSE;
MatFactorInfo finfo;
PetscInitialize(&argc,&argv,(char*) 0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
/* Get local dimensions of matrices */
ierr = PetscOptionsGetInt(NULL,"-m",&m,NULL);CHKERRQ(ierr);
n = m;
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
p = m/2;
ierr = PetscOptionsGetInt(NULL,"-p",&p,NULL);CHKERRQ(ierr);
ierr = PetscOptionsHasName(NULL,"-mats_view",&mats_view);CHKERRQ(ierr);
/* Create matrix A */
ierr = PetscPrintf(PETSC_COMM_WORLD," Create Elemental matrix A\n");CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(A,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/* Set local matrix entries */
ierr = MatGetOwnershipIS(A,&isrows,&iscols);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr);
ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr);
ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(nrows*ncols,&v);CHKERRQ(ierr);
for (i=0; i<nrows; i++) {
for (j=0; j<ncols; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
v[i*ncols+j] = rval;
}
}
ierr = MatSetValues(A,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr);
ierr = ISDestroy(&isrows);CHKERRQ(ierr);
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
ierr = PetscFree(v);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "A: nrows %d, m %d; ncols %d, n %d\n",nrows,m,ncols,n);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create rhs matrix B */
ierr = PetscPrintf(PETSC_COMM_WORLD," Create rhs matrix B\n");CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&B);CHKERRQ(ierr);
ierr = MatSetSizes(B,m,p,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(B,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(B);CHKERRQ(ierr);
ierr = MatSetUp(B);CHKERRQ(ierr);
ierr = MatGetOwnershipIS(B,&isrows,&iscols);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr);
ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr);
ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(nrows*ncols,&v);CHKERRQ(ierr);
for (i=0; i<nrows; i++) {
for (j=0; j<ncols; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
v[i*ncols+j] = rval;
}
}
ierr = MatSetValues(B,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr);
ierr = ISDestroy(&isrows);CHKERRQ(ierr);
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
ierr = PetscFree(v);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "B: nrows %d, m %d; ncols %d, p %d\n",nrows,m,ncols,p);CHKERRQ(ierr);
ierr = MatView(B,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create rhs vector b and solution x (same size as b) */
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
ierr = VecGetArray(b,&barray);CHKERRQ(ierr);
for (j=0; j<m; j++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
barray[j] = rval;
}
ierr = VecRestoreArray(b,&barray);CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d] b: m %d\n",rank,m);CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
ierr = VecView(b,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
/* Create matrix X - same size as B */
ierr = PetscPrintf(PETSC_COMM_WORLD," Create solution matrix X\n");CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&X);CHKERRQ(ierr);
ierr = MatSetSizes(X,m,p,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(X,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(X);CHKERRQ(ierr);
ierr = MatSetUp(X);CHKERRQ(ierr);
ierr = MatAssemblyBegin(X,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(X,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Cholesky factorization */
/*------------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD," Create Elemental matrix Aher\n");CHKERRQ(ierr);
ierr = MatHermitianTranspose(A,MAT_INITIAL_MATRIX,&Aher);CHKERRQ(ierr);
ierr = MatAXPY(Aher,1.0,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr); /* Aher = A + A^T */
if (!rank) { /* add 100.0 to diagonals of Aher to make it spd */
PetscInt M,N;
ierr = MatGetSize(Aher,&M,&N);CHKERRQ(ierr);
for (i=0; i<M; i++) {
rval = 100.0;
ierr = MatSetValues(Aher,1,&i,1,&i,&rval,ADD_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(Aher,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Aher,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Aher:\n");CHKERRQ(ierr);
ierr = MatView(Aher,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Cholesky factorization */
/*------------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD," Test Cholesky Solver \n");CHKERRQ(ierr);
/* In-place Cholesky */
/* Create matrix factor G, then copy Aher to G */
ierr = MatCreate(PETSC_COMM_WORLD,&G);CHKERRQ(ierr);
ierr = MatSetSizes(G,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(G,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(G);CHKERRQ(ierr);
ierr = MatSetUp(G);CHKERRQ(ierr);
ierr = MatAssemblyBegin(G,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(G,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCopy(Aher,G,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
/* Only G = U^T * U is implemented for now */
ierr = MatCholeskyFactor(G,0,0);CHKERRQ(ierr);
if (mats_view) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "Cholesky Factor G:\n");CHKERRQ(ierr);
ierr = MatView(G,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Solve U^T * U x = b and U^T * U X = B */
ierr = MatSolve(G,b,x);CHKERRQ(ierr);
ierr = MatMatSolve(G,B,X);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
/* Out-place Cholesky */
ierr = MatGetFactor(Aher,MATSOLVERELEMENTAL,MAT_FACTOR_CHOLESKY,&G);CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(G,Aher,0,&finfo);CHKERRQ(ierr);
ierr = MatCholeskyFactorNumeric(G,Aher,&finfo);CHKERRQ(ierr);
if (mats_view) {
ierr = MatView(G,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = MatSolve(G,b,x);CHKERRQ(ierr);
ierr = MatMatSolve(G,B,X);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
/* Check norm(Aher*x - b) */
ierr = VecCreate(PETSC_COMM_WORLD,&c);CHKERRQ(ierr);
ierr = VecSetSizes(c,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(c);CHKERRQ(ierr);
ierr = MatMult(Aher,x,c);CHKERRQ(ierr);
ierr = VecAXPY(c,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(c,NORM_1,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |Aher*x - b| for Cholesky %G\n",norm);CHKERRQ(ierr);
}
/* Check norm(Aher*X - B) */
ierr = MatMatMult(Aher,X,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&C);CHKERRQ(ierr);
ierr = MatAXPY(C,-1.0,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatNorm(C,NORM_1,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |Aher*X - B| for Cholesky %G\n",norm);CHKERRQ(ierr);
}
/* LU factorization */
/*------------------*/
ierr = PetscPrintf(PETSC_COMM_WORLD," Test LU Solver \n");CHKERRQ(ierr);
/* In-place LU */
/* Create matrix factor F, then copy A to F */
ierr = MatCreate(PETSC_COMM_WORLD,&F);CHKERRQ(ierr);
ierr = MatSetSizes(F,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetType(F,MATELEMENTAL);CHKERRQ(ierr);
ierr = MatSetFromOptions(F);CHKERRQ(ierr);
ierr = MatSetUp(F);CHKERRQ(ierr);
ierr = MatAssemblyBegin(F,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(F,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCopy(A,F,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
/* Create vector d to test MatSolveAdd() */
ierr = VecDuplicate(x,&d);CHKERRQ(ierr);
ierr = VecCopy(x,d);CHKERRQ(ierr);
/* PF=LU or F=LU factorization - perms is ignored by Elemental;
set finfo.dtcol !0 or 0 to enable/disable partial pivoting */
finfo.dtcol = 0.1;
ierr = MatLUFactor(F,0,0,&finfo);CHKERRQ(ierr);
/* Solve LUX = PB or LUX = B */
ierr = MatSolveAdd(F,b,d,x);CHKERRQ(ierr);
ierr = MatMatSolve(F,B,X);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
/* Check norm(A*X - B) */
ierr = VecCreate(PETSC_COMM_WORLD,&e);CHKERRQ(ierr);
ierr = VecSetSizes(e,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(e);CHKERRQ(ierr);
ierr = MatMult(A,x,c);CHKERRQ(ierr);
ierr = MatMult(A,d,e);CHKERRQ(ierr);
ierr = VecAXPY(c,-1.0,e);CHKERRQ(ierr);
ierr = VecAXPY(c,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(c,NORM_1,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |A*x - b| for LU %G\n",norm);CHKERRQ(ierr);
}
ierr = MatMatMult(A,X,MAT_REUSE_MATRIX,PETSC_DEFAULT,&C);CHKERRQ(ierr);
ierr = MatAXPY(C,-1.0,B,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatNorm(C,NORM_1,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Warning: |A*X - B| for LU %G\n",norm);CHKERRQ(ierr);
}
/* Out-place LU */
ierr = MatGetFactor(A,MATSOLVERELEMENTAL,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
ierr = MatLUFactorSymbolic(F,A,0,0,&finfo);CHKERRQ(ierr);
ierr = MatLUFactorNumeric(F,A,&finfo);CHKERRQ(ierr);
if (mats_view) {
ierr = MatView(F,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = MatSolve(F,b,x);CHKERRQ(ierr);
ierr = MatMatSolve(F,B,X);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
/* Free space */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&Aher);CHKERRQ(ierr);
ierr = MatDestroy(&B);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&c);CHKERRQ(ierr);
ierr = VecDestroy(&d);CHKERRQ(ierr);
ierr = VecDestroy(&e);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat C;
PetscInt i,j,m = 5,n = 2,Ii,J;
PetscErrorCode ierr;
PetscMPIInt rank,size;
PetscScalar v;
Vec x,y;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
n = 2*size;
/* Create the matrix for the five point stencil, YET AGAIN */
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
for (i=0; i<m; i++) {
for (j=2*rank; j<2*rank+2; j++) {
v = -1.0; Ii = j + n*i;
if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
for (i=0; i<m; i++) {
for (j=2*rank; j<2*rank+2; j++) {
v = 1.0; Ii = j + n*i;
if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);}
v = -4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr);
}
}
/* Introduce new nonzero that requires new construction for
matrix-vector product */
if (rank) {
Ii = rank-1; J = m*n-1;
ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Form a couple of vectors to test matrix-vector product */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,m*n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
v = 1.0; ierr = VecSet(x,v);CHKERRQ(ierr);
ierr = MatMult(C,x,y);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode PipesView(Vec X,DM networkdm,Wash wash)
{
PetscErrorCode ierr;
Pipe pipe;
DMNetworkComponentGenericDataType *nwarr;
PetscInt pipeOffset,key,Start,End;
PetscMPIInt rank;
PetscInt nx,nnodes,nidx,*idx1,*idx2,*idx1_h,*idx2_h,idx_start,i,k,k1,xstart,j1;
Vec Xq,Xh,localX;
IS is1_q,is2_q,is1_h,is2_h;
VecScatter ctx_q,ctx_h;
PetscFunctionBegin;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* get num of local and global total nnodes */
nidx = wash->nnodes_loc;
ierr = MPIU_Allreduce(&nidx,&nx,1,MPIU_INT,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&Xq);CHKERRQ(ierr);
if (rank == 0) { /* all entries of Xq are in proc[0] */
ierr = VecSetSizes(Xq,nx,PETSC_DECIDE);CHKERRQ(ierr);
} else {
ierr = VecSetSizes(Xq,0,PETSC_DECIDE);CHKERRQ(ierr);
}
ierr = VecSetFromOptions(Xq);CHKERRQ(ierr);
ierr = VecSet(Xq,0.0);CHKERRQ(ierr);
ierr = VecDuplicate(Xq,&Xh);CHKERRQ(ierr);
ierr = DMGetLocalVector(networkdm,&localX);CHKERRQ(ierr);
/* set idx1 and idx2 */
ierr = PetscCalloc4(nidx,&idx1,nidx,&idx2,nidx,&idx1_h,nidx,&idx2_h);CHKERRQ(ierr);
ierr = DMNetworkGetComponentDataArray(networkdm,&nwarr);CHKERRQ(ierr);
ierr = DMNetworkGetEdgeRange(networkdm,&Start, &End);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(X,&xstart,NULL);CHKERRQ(ierr);
k1 = 0;
j1 = 0;
for (i = Start; i < End; i++) {
ierr = DMNetworkGetComponentTypeOffset(networkdm,i,0,&key,&pipeOffset);CHKERRQ(ierr);
pipe = (Pipe)(nwarr+pipeOffset);
nnodes = pipe->nnodes;
idx_start = pipe->id*nnodes;
for (k=0; k<nnodes; k++) {
idx1[k1] = xstart + j1*2*nnodes + 2*k;
idx2[k1] = idx_start + k;
idx1_h[k1] = xstart + j1*2*nnodes + 2*k + 1;
idx2_h[k1] = idx_start + k;
k1++;
}
j1++;
}
ierr = ISCreateGeneral(PETSC_COMM_SELF,nidx,idx1,PETSC_COPY_VALUES,&is1_q);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,nidx,idx2,PETSC_COPY_VALUES,&is2_q);CHKERRQ(ierr);
ierr = VecScatterCreate(X,is1_q,Xq,is2_q,&ctx_q);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx_q,X,Xq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx_q,X,Xq,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,nidx,idx1_h,PETSC_COPY_VALUES,&is1_h);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,nidx,idx2_h,PETSC_COPY_VALUES,&is2_h);CHKERRQ(ierr);
ierr = VecScatterCreate(X,is1_h,Xh,is2_h,&ctx_h);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx_h,X,Xh,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx_h,X,Xh,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (!rank) printf("Xq: \n");
ierr = VecView(Xq,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
if (!rank) printf("Xh: \n");
ierr = VecView(Xh,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx_q);CHKERRQ(ierr);
ierr = PetscFree4(idx1,idx2,idx1_h,idx2_h);CHKERRQ(ierr);
ierr = ISDestroy(&is1_q);CHKERRQ(ierr);
ierr = ISDestroy(&is2_q);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx_h);CHKERRQ(ierr);
ierr = ISDestroy(&is1_h);CHKERRQ(ierr);
ierr = ISDestroy(&is2_h);CHKERRQ(ierr);
ierr = VecDestroy(&Xq);CHKERRQ(ierr);
ierr = VecDestroy(&Xh);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(networkdm,&localX);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int
PetscSOE::setSizeParallel(int newSize, int N,
int d_nz, int *d_nnz,
int o_nz, int *o_nnz)
{
int result = 0;
int oldSize = size;
size = newSize;
isFactored = 0;
// delete the old
if (B != 0) delete [] B;
if (X != 0) delete [] X;
// invoke the petsc destructors
if (A != 0) MatDestroy(A);
if (b != 0) VecDestroy(b);
if (x != 0) VecDestroy(x);
// create the new
B = new double[size];
X = new double[size];
if (B == 0 || X == 0) {
opserr << "WARNING PetscSOE::PetscSOE :";
opserr << " ran out of memory for vectors (size) (";
opserr << size << ") \n";
size = 0;
result = -1;
}
// zero the vectors
for (int j=0; j<size; j++) {
B[j] = 0;
X[j] = 0;
}
if (vectX != 0)
delete vectX;
if (vectB != 0)
delete vectB;
vectX = new Vector(X,size);
vectB = new Vector(B,size);
// invoke the petsc constructors
int ierr = OptionsGetInt(PETSC_NULL,"-n",&size,&flg); CHKERRA(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,size,size,&A); CHKERRA(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,PETSC_DECIDE,size,&x); CHKERRA(ierr);
ierr = VecDuplicate(x,&b); CHKERRA(ierr);
// invoke setSize() on the Solver
int solverOK = theSolver->setSize();
if (solverOK < 0) {
opserr << "WARNING:PetscSOE::setSize :";
opserr << " solver failed setSize()\n";
return solverOK;
}
return result;
}
int main(int argc,char **argv)
{
TS ts; /* ODE integrator */
Vec U; /* solution will be stored here */
PetscErrorCode ierr;
PetscMPIInt size;
PetscInt n = 2;
PetscScalar *u;
AppCtx app;
PetscInt direction[2];
PetscBool terminate[2];
PetscBool rhs_form=PETSC_FALSE,hist=PETSC_TRUE;
TSAdapt adapt;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Initialize program
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size > 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Only for sequential runs");
app.nbounces = 0;
app.maxbounces = 10;
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"ex40 options","");CHKERRQ(ierr);
ierr = PetscOptionsInt("-maxbounces","","",app.maxbounces,&app.maxbounces,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-test_adapthistory","","",hist,&hist,NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create timestepping solver context
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr);
ierr = TSSetType(ts,TSROSW);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set ODE routines
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr);
/* Users are advised against the following branching and code duplication.
For problems without a mass matrix like the one at hand, the RHSFunction
(and companion RHSJacobian) interface is enough to support both explicit
and implicit timesteppers. This tutorial example also deals with the
IFunction/IJacobian interface for demonstration and testing purposes. */
ierr = PetscOptionsGetBool(NULL,NULL,"-rhs-form",&rhs_form,NULL);CHKERRQ(ierr);
if (rhs_form) {
ierr = TSSetRHSFunction(ts,NULL,RHSFunction,NULL);CHKERRQ(ierr);
ierr = TSSetRHSJacobian(ts,NULL,NULL,RHSJacobian,NULL);CHKERRQ(ierr);
} else {
Mat A; /* Jacobian matrix */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(A,MATDENSE);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = TSSetIFunction(ts,NULL,IFunction,NULL);CHKERRQ(ierr);
ierr = TSSetIJacobian(ts,A,A,IJacobian,NULL);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set initial conditions
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = VecCreate(PETSC_COMM_WORLD,&U);CHKERRQ(ierr);
ierr = VecSetSizes(U,n,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetUp(U);CHKERRQ(ierr);
ierr = VecGetArray(U,&u);CHKERRQ(ierr);
u[0] = 0.0;
u[1] = 20.0;
ierr = VecRestoreArray(U,&u);CHKERRQ(ierr);
ierr = TSSetSolution(ts,U);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set solver options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSSetSaveTrajectory(ts);CHKERRQ(ierr);
ierr = TSSetMaxTime(ts,30.0);CHKERRQ(ierr);
ierr = TSSetExactFinalTime(ts,TS_EXACTFINALTIME_STEPOVER);CHKERRQ(ierr);
ierr = TSSetTimeStep(ts,0.1);CHKERRQ(ierr);
/* The adapative time step controller could take very large timesteps resulting in
the same event occuring multiple times in the same interval. A maximum step size
limit is enforced here to avoid this issue. */
ierr = TSGetAdapt(ts,&adapt);CHKERRQ(ierr);
ierr = TSAdaptSetStepLimits(adapt,0.0,0.5);CHKERRQ(ierr);
/* Set directions and terminate flags for the two events */
direction[0] = -1; direction[1] = -1;
terminate[0] = PETSC_FALSE; terminate[1] = PETSC_TRUE;
ierr = TSSetEventHandler(ts,2,direction,terminate,EventFunction,PostEventFunction,(void*)&app);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Run timestepping solver
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSSolve(ts,U);CHKERRQ(ierr);
if (hist) { /* replay following history */
TSTrajectory tj;
PetscReal tf,t0,dt;
app.nbounces = 0;
ierr = TSGetTime(ts,&tf);CHKERRQ(ierr);
ierr = TSSetMaxTime(ts,tf);CHKERRQ(ierr);
ierr = TSSetStepNumber(ts,0);CHKERRQ(ierr);
ierr = TSRestartStep(ts);CHKERRQ(ierr);
ierr = TSSetExactFinalTime(ts,TS_EXACTFINALTIME_MATCHSTEP);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
ierr = TSGetAdapt(ts,&adapt);CHKERRQ(ierr);
ierr = TSAdaptSetType(adapt,TSADAPTHISTORY);CHKERRQ(ierr);
ierr = TSGetTrajectory(ts,&tj);CHKERRQ(ierr);
ierr = TSAdaptHistorySetTrajectory(adapt,tj,PETSC_FALSE);CHKERRQ(ierr);
ierr = TSAdaptHistoryGetStep(adapt,0,&t0,&dt);CHKERRQ(ierr);
/* this example fails with single (or smaller) precision */
#if defined(PETSC_USE_REAL_SINGLE) || defined(PETSC_USE_REAL__FP16)
ierr = TSAdaptSetType(adapt,TSADAPTBASIC);CHKERRQ(ierr);
ierr = TSAdaptSetStepLimits(adapt,0.0,0.5);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
#endif
ierr = TSSetTime(ts,t0);CHKERRQ(ierr);
ierr = TSSetTimeStep(ts,dt);CHKERRQ(ierr);
ierr = TSResetTrajectory(ts);CHKERRQ(ierr);
ierr = VecGetArray(U,&u);CHKERRQ(ierr);
u[0] = 0.0;
u[1] = 20.0;
ierr = VecRestoreArray(U,&u);CHKERRQ(ierr);
ierr = TSSolve(ts,U);CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free work space. All PETSc objects should be destroyed when they are no longer needed.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = VecDestroy(&U);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
SNES snes; /* SNES context */
Vec x,r,F; /* vectors */
Mat J; /* Jacobian */
PetscErrorCode ierr;
PetscInt it,n = 11,i;
PetscReal h,xp = 0.0;
PetscScalar v;
const PetscScalar a = X0DOT;
const PetscScalar b = X1;
const PetscScalar k = KPOW;
PetscScalar v2;
PetscScalar *xx;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,NULL,"-second_order",&second_order,NULL);CHKERRQ(ierr);
h = 1.0/(n-1);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create nonlinear solver context
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = SNESCreate(PETSC_COMM_WORLD,&snes);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create vector data structures; set function evaluation routine
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = VecCreate(PETSC_COMM_SELF,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&r);CHKERRQ(ierr);
ierr = VecDuplicate(x,&F);CHKERRQ(ierr);
ierr = SNESSetFunction(snes,r,FormFunction,(void*)F);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create matrix data structures; set Jacobian evaluation routine
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = MatCreateSeqAIJ(PETSC_COMM_SELF,n,n,3,NULL,&J);CHKERRQ(ierr);
/*
Note that in this case we create separate matrices for the Jacobian
and preconditioner matrix. Both of these are computed in the
routine FormJacobian()
*/
/* ierr = SNESSetJacobian(snes,NULL,JPrec,FormJacobian,0);CHKERRQ(ierr); */
ierr = SNESSetJacobian(snes,J,J,FormJacobian,0);CHKERRQ(ierr);
/* ierr = SNESSetJacobian(snes,J,JPrec,FormJacobian,0);CHKERRQ(ierr); */
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Customize nonlinear solver; set runtime options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Initialize application:
Store right-hand-side of PDE and exact solution
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* set right hand side and initial guess to be exact solution of continuim problem */
#define SQR(x) ((x)*(x))
xp = 0.0;
for (i=0; i<n; i++)
{
v = k*(k-1.)*(b-a)*PetscPowScalar(xp,k-2.) + SQR(a*xp) + SQR(b-a)*PetscPowScalar(xp,2.*k) + 2.*a*(b-a)*PetscPowScalar(xp,k+1.);
ierr = VecSetValues(F,1,&i,&v,INSERT_VALUES);CHKERRQ(ierr);
v2 = a*xp + (b-a)*PetscPowScalar(xp,k);
ierr = VecSetValues(x,1,&i,&v2,INSERT_VALUES);CHKERRQ(ierr);
xp += h;
}
/* perturb initial guess */
ierr = VecGetArray(x,&xx);
for (i=0; i<n; i++) {
v2 = xx[i]*sperturb;
ierr = VecSetValues(x,1,&i,&v2,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecRestoreArray(x,&xx);CHKERRQ(ierr);
ierr = SNESSolve(snes,NULL,x);CHKERRQ(ierr);
ierr = SNESGetIterationNumber(snes,&it);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"SNES iterations = %D\n\n",it);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&r);CHKERRQ(ierr);
ierr = VecDestroy(&F);CHKERRQ(ierr); ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
/*@
DMPlexCreateGmsh - Create a DMPlex mesh from a Gmsh file viewer
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. viewer - The Viewer associated with a Gmsh file
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.geuz.org/gmsh/doc/texinfo/#MSH-ASCII-file-format
and http://www.geuz.org/gmsh/doc/texinfo/#MSH-binary-file-format
Level: beginner
.keywords: mesh,Gmsh
.seealso: DMPLEX, DMCreate()
@*/
PetscErrorCode DMPlexCreateGmsh(MPI_Comm comm, PetscViewer viewer, PetscBool interpolate, DM *dm)
{
PetscViewerType vtype;
GmshElement *gmsh_elem;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords, *coordsIn = NULL;
PetscInt dim = 0, coordSize, c, v, d, r, cell;
int i, numVertices = 0, numCells = 0, trueNumCells = 0, numRegions = 0, snum;
PetscMPIInt num_proc, rank;
char line[PETSC_MAX_PATH_LEN];
PetscBool match, binary, bswap = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
ierr = PetscViewerGetType(viewer, &vtype);CHKERRQ(ierr);
ierr = PetscStrcmp(vtype, PETSCVIEWERBINARY, &binary);CHKERRQ(ierr);
if (!rank || binary) {
PetscBool match;
int fileType, dataSize;
float version;
/* Read header */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$MeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 3, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%f %d %d", &version, &fileType, &dataSize);
if (snum != 3) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse Gmsh file header: %s", line);
if (version < 2.0) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Gmsh file must be at least version 2.0");
if (dataSize != sizeof(double)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Data size %d is not valid for a Gmsh file", dataSize);
if (binary) {
int checkInt;
ierr = PetscViewerRead(viewer, &checkInt, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
if (checkInt != 1) {
ierr = PetscByteSwap(&checkInt, PETSC_ENUM, 1);CHKERRQ(ierr);
if (checkInt == 1) bswap = PETSC_TRUE;
else SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh binary file", fileType);
}
} else if (fileType) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File type %d is not a valid Gmsh ASCII file", fileType);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndMeshFormat", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* OPTIONAL Read physical names */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$PhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (match) {
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numRegions);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
for (r = 0; r < numRegions; ++r) {
PetscInt rdim, tag;
ierr = PetscViewerRead(viewer, &rdim, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &tag, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
ierr = PetscStrncmp(line, "$EndPhysicalNames", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Initial read for vertex section */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
}
/* Read vertices */
ierr = PetscStrncmp(line, "$Nodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);
snum = sscanf(line, "%d", &numVertices);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscMalloc1(numVertices*3, &coordsIn);CHKERRQ(ierr);
if (binary) {
size_t doubleSize, intSize;
PetscInt elementSize;
char *buffer;
PetscScalar *baseptr;
ierr = PetscDataTypeGetSize(PETSC_ENUM, &intSize);CHKERRQ(ierr);
ierr = PetscDataTypeGetSize(PETSC_DOUBLE, &doubleSize);CHKERRQ(ierr);
elementSize = (intSize + 3*doubleSize);
ierr = PetscMalloc1(elementSize*numVertices, &buffer);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, buffer, elementSize*numVertices, NULL, PETSC_CHAR);CHKERRQ(ierr);
if (bswap) ierr = PetscByteSwap(buffer, PETSC_CHAR, elementSize*numVertices);CHKERRQ(ierr);
for (v = 0; v < numVertices; ++v) {
baseptr = ((PetscScalar*)(buffer+v*elementSize+intSize));
coordsIn[v*3+0] = baseptr[0];
coordsIn[v*3+1] = baseptr[1];
coordsIn[v*3+2] = baseptr[2];
}
ierr = PetscFree(buffer);CHKERRQ(ierr);
} else {
for (v = 0; v < numVertices; ++v) {
ierr = PetscViewerRead(viewer, &i, 1, NULL, PETSC_ENUM);CHKERRQ(ierr);
ierr = PetscViewerRead(viewer, &(coordsIn[v*3]), 3, NULL, PETSC_DOUBLE);CHKERRQ(ierr);
if (i != (int)v+1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid node number %d should be %d", i, v+1);
}
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndNodes", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
/* Read cells */
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$Elements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
snum = sscanf(line, "%d", &numCells);
if (snum != 1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
if (!rank || binary) {
/* Gmsh elements can be of any dimension/co-dimension, so we need to traverse the
file contents multiple times to figure out the true number of cells and facets
in the given mesh. To make this more efficient we read the file contents only
once and store them in memory, while determining the true number of cells. */
ierr = DMPlexCreateGmsh_ReadElement(viewer, numCells, binary, bswap, &gmsh_elem);CHKERRQ(ierr);
for (trueNumCells=0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim > dim) {dim = gmsh_elem[c].dim; trueNumCells = 0;}
if (gmsh_elem[c].dim == dim) trueNumCells++;
}
ierr = PetscViewerRead(viewer, line, 1, NULL, PETSC_STRING);CHKERRQ(ierr);;
ierr = PetscStrncmp(line, "$EndElements", PETSC_MAX_PATH_LEN, &match);CHKERRQ(ierr);
if (!match) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "File is not a valid Gmsh file");
}
/* For binary we read on all ranks, but only build the plex on rank 0 */
if (binary && rank) {trueNumCells = 0; numVertices = 0;};
/* Allocate the cell-vertex mesh */
ierr = DMPlexSetChart(*dm, 0, trueNumCells+numVertices);CHKERRQ(ierr);
if (!rank) {
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
ierr = DMPlexSetConeSize(*dm, cell, gmsh_elem[c].numNodes);CHKERRQ(ierr);
cell++;
}
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
/* Add cell-vertex connections */
if (!rank) {
PetscInt pcone[8], corner;
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + trueNumCells-1;
}
if (dim == 3) {
/* Tetrahedra are inverted */
if (gmsh_elem[c].numNodes == 4) {
PetscInt tmp = pcone[0];
pcone[0] = pcone[1];
pcone[1] = tmp;
}
/* Hexahedra are inverted */
if (gmsh_elem[c].numNodes == 8) {
PetscInt tmp = pcone[1];
pcone[1] = pcone[3];
pcone[3] = tmp;
}
}
ierr = DMPlexSetCone(*dm, cell, pcone);CHKERRQ(ierr);
cell++;
}
}
}
ierr = MPI_Bcast(&dim, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
if (!rank) {
/* Apply boundary IDs by finding the relevant facets with vertex joins */
PetscInt pcone[8], corner, vStart, vEnd;
ierr = DMPlexGetDepthStratum(*dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
for (c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim-1) {
PetscInt joinSize;
const PetscInt *join;
for (corner = 0; corner < gmsh_elem[c].numNodes; ++corner) {
pcone[corner] = gmsh_elem[c].nodes[corner] + vStart - 1;
}
ierr = DMPlexGetFullJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
if (joinSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not determine Plex facet for element %d", gmsh_elem[c].id);
ierr = DMSetLabelValue(*dm, "Face Sets", join[0], gmsh_elem[c].tags[0]);CHKERRQ(ierr);
ierr = DMPlexRestoreJoin(*dm, gmsh_elem[c].numNodes, (const PetscInt *) pcone, &joinSize, &join);CHKERRQ(ierr);
}
}
/* Create cell sets */
for (cell = 0, c = 0; c < numCells; ++c) {
if (gmsh_elem[c].dim == dim) {
if (gmsh_elem[c].numTags > 0) {
ierr = DMSetLabelValue(*dm, "Cell Sets", cell, gmsh_elem[c].tags[0]);CHKERRQ(ierr);
cell++;
}
}
}
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, trueNumCells, trueNumCells + numVertices);CHKERRQ(ierr);
for (v = trueNumCells; v < trueNumCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetBlockSize(coordinates, dim);CHKERRQ(ierr);
ierr = VecSetType(coordinates, VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
for (v = 0; v < numVertices; ++v) {
for (d = 0; d < dim; ++d) {
coords[v*dim+d] = coordsIn[v*3+d];
}
}
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = PetscFree(coordsIn);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Clean up intermediate storage */
if (!rank || binary) ierr = PetscFree(gmsh_elem);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_CreateGmsh,*dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,RHS,C,F,X;
Vec u,x,b;
PetscErrorCode ierr;
PetscMPIInt rank,nproc;
PetscInt i,m,n,nfact,nsolve,nrhs,ipack=0;
PetscScalar *array,rval;
PetscReal norm,tol=1.e-12;
IS perm,iperm;
MatFactorInfo info;
PetscRandom rand;
PetscBool flg,testMatSolve=PETSC_TRUE,testMatMatSolve=PETSC_TRUE;
PetscViewer fd; /* viewer */
char file[PETSC_MAX_PATH_LEN]; /* input file name */
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &nproc);CHKERRQ(ierr);
/* Determine file from which we read the matrix A */
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,1,"Must indicate binary file with the -f option");
/* Load matrix A */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
if (m != n) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ, "This example is not intended for rectangular matrices (%d, %d)", m, n);
/* Create dense matrix C and X; C holds true solution with identical colums */
nrhs = 2;
ierr = PetscOptionsGetInt(NULL,"-nrhs",&nrhs,NULL);CHKERRQ(ierr);
if (!rank) printf("ex125: nrhs %d\n",nrhs);
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,m,PETSC_DECIDE,PETSC_DECIDE,nrhs);CHKERRQ(ierr);
ierr = MatSetType(C,MATDENSE);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rand);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rand);CHKERRQ(ierr);
ierr = MatSetRandom(C,rand);CHKERRQ(ierr);
ierr = MatDuplicate(C,MAT_DO_NOT_COPY_VALUES,&X);CHKERRQ(ierr);
/* Create vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);CHKERRQ(ierr); /* save the true solution */
/* Test LU Factorization */
ierr = MatGetOrdering(A,MATORDERINGND,&perm,&iperm);CHKERRQ(ierr);
/*ierr = ISView(perm,PETSC_VIEWER_STDOUT_WORLD);*/
/*ierr = ISView(perm,PETSC_VIEWER_STDOUT_SELF);*/
ierr = PetscOptionsGetInt(NULL,"-mat_solver_package",&ipack,NULL);CHKERRQ(ierr);
switch (ipack) {
case 0:
#if defined(PETSC_HAVE_SUPERLU)
if (!rank) printf(" SUPERLU LU:\n");
ierr = MatGetFactor(A,MATSOLVERSUPERLU,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
break;
#endif
case 1:
#if defined(PETSC_HAVE_SUPERLU_DIST)
if (!rank) printf(" SUPERLU_DIST LU:\n");
ierr = MatGetFactor(A,MATSOLVERSUPERLU_DIST,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
break;
#endif
case 2:
#if defined(PETSC_HAVE_MUMPS)
if (!rank) printf(" MUMPS LU:\n");
ierr = MatGetFactor(A,MATSOLVERMUMPS,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
{
/* test mumps options */
PetscInt icntl_7 = 5;
ierr = MatMumpsSetIcntl(F,7,icntl_7);CHKERRQ(ierr);
}
break;
#endif
default:
if (!rank) printf(" PETSC LU:\n");
ierr = MatGetFactor(A,MATSOLVERPETSC,MAT_FACTOR_LU,&F);CHKERRQ(ierr);
}
info.fill = 5.0;
ierr = MatLUFactorSymbolic(F,A,perm,iperm,&info);CHKERRQ(ierr);
for (nfact = 0; nfact < 2; nfact++) {
if (!rank) printf(" %d-the LU numfactorization \n",nfact);
ierr = MatLUFactorNumeric(F,A,&info);CHKERRQ(ierr);
/* Test MatMatSolve() */
/*
if ((ipack == 0 || ipack == 2) && testMatMatSolve) {
printf(" MatMatSolve() is not implemented for this package. Skip the testing.\n");
testMatMatSolve = PETSC_FALSE;
}
*/
if (testMatMatSolve) {
if (!nfact) {
ierr = MatMatMult(A,C,MAT_INITIAL_MATRIX,2.0,&RHS);CHKERRQ(ierr);
} else {
ierr = MatMatMult(A,C,MAT_REUSE_MATRIX,2.0,&RHS);CHKERRQ(ierr);
}
for (nsolve = 0; nsolve < 2; nsolve++) {
if (!rank) printf(" %d-the MatMatSolve \n",nsolve);
ierr = MatMatSolve(F,RHS,X);CHKERRQ(ierr);
/* Check the error */
ierr = MatAXPY(X,-1.0,C,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatNorm(X,NORM_FROBENIUS,&norm);CHKERRQ(ierr);
if (norm > tol) {
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"1st MatMatSolve: Norm of error %g, nsolve %d\n",norm,nsolve);CHKERRQ(ierr);
}
}
}
}
/* Test MatSolve() */
if (testMatSolve) {
for (nsolve = 0; nsolve < 2; nsolve++) {
ierr = VecGetArray(x,&array);CHKERRQ(ierr);
for (i=0; i<m; i++) {
ierr = PetscRandomGetValue(rand,&rval);CHKERRQ(ierr);
array[i] = rval;
}
ierr = VecRestoreArray(x,&array);CHKERRQ(ierr);
ierr = VecCopy(x,u);CHKERRQ(ierr);
ierr = MatMult(A,x,b);CHKERRQ(ierr);
if (!rank) printf(" %d-the MatSolve \n",nsolve);
ierr = MatSolve(F,b,x);CHKERRQ(ierr);
/* Check the error */
ierr = VecAXPY(u,-1.0,x);CHKERRQ(ierr); /* u <- (-1.0)x + u */
ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr);
if (norm > tol) {
ierr = MatMult(A,x,u);CHKERRQ(ierr); /* u = A*x */
PetscReal resi;
ierr = VecAXPY(u,-1.0,b);CHKERRQ(ierr); /* u <- (-1.0)b + u */
ierr = VecNorm(u,NORM_2,&resi);CHKERRQ(ierr);
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatSolve: Norm of error %g, resi %g, LU numfact %d\n",norm,resi,nfact);CHKERRQ(ierr);
}
}
}
}
}
/* Free data structures */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
ierr = MatDestroy(&X);CHKERRQ(ierr);
if (testMatMatSolve) {
ierr = MatDestroy(&RHS);CHKERRQ(ierr);
}
ierr = PetscRandomDestroy(&rand);CHKERRQ(ierr);
ierr = ISDestroy(&perm);CHKERRQ(ierr);
ierr = ISDestroy(&iperm);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc, char** argv)
{
/* the total number of grid points in each spatial direction is (n+1) */
/* the total number of degrees-of-freedom in each spatial direction is (n-1) */
/* this version requires n to be a power of 2 */
if( argc < 2 ) {
printf("need a problem size\n");
return 1;
}
int n = atoi(argv[1]);
int m = n-1;
// Initialize Petsc
PetscInitialize(&argc,&argv,0,PETSC_NULL);
// Create our vector
Vec b;
VecCreate(PETSC_COMM_WORLD,&b);
VecSetSizes(b,PETSC_DECIDE,m*m);
VecSetFromOptions(b);
// Create our matrix
Mat A;
MatCreate(PETSC_COMM_WORLD,&A);
MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,m*m,m*m);
MatSetUp(A);
// Create linear solver object
KSP ksp;
KSPCreate(PETSC_COMM_WORLD,&ksp);
// setup rhs
double h = 1./(double)n;
for (int j=0; j < m*m; j++)
VecSetValue(b,j,h*h,INSERT_VALUES);
// setup matrix
for (int i=0;i<m*m;++i) {
MatSetValue(A,i,i,4.f,INSERT_VALUES);
if (i%m != m-1)
MatSetValue(A,i,i+1,-1.f,INSERT_VALUES);
if (i%m)
MatSetValue(A,i,i-1,-1.f,INSERT_VALUES);
if (i > m)
MatSetValue(A,i,i-m,-1.f,INSERT_VALUES);
if (i < m*(m-1))
MatSetValue(A,i,i+m,-1.f,INSERT_VALUES);
}
// sync processes
VecAssemblyBegin(b);
VecAssemblyEnd(b);
MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
// solve
KSPSetType(ksp,"cg");
KSPSetTolerances(ksp,1.e-10,1.e-10,1.e6,10000);
KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);
PC pc;
KSPGetPC(ksp,&pc);
PCSetFromOptions(pc);
PCSetUp(pc);
// setup solver
KSPSetFromOptions(ksp);
KSPSetUp(ksp);
Vec x;
VecDuplicate(b,&x);
KSPSolve(ksp,b,x);
double val;
VecNorm(x,NORM_INFINITY,&val);
printf (" umax = %e \n",val);
PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Vec x,b,u;
Mat A; /* linear system matrix */
KSP ksp; /* linear solver context */
PetscRandom rctx; /* random number generator context */
PetscReal norm; /* norm of solution error */
PetscInt i,j,k,l,n = 27,its,bs = 2,Ii,J;
PetscErrorCode ierr;
PetscScalar v;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,n*bs,n*bs,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetBlockSize(A,bs);CHKERRQ(ierr);
ierr = MatSetType(A,MATSEQBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/*
Don't bother to preallocate matrix
*/
ierr = PetscRandomCreate(PETSC_COMM_SELF,&rctx);CHKERRQ(ierr);
for (i=0; i<n; i++) {
for (j=0; j<n; j++) {
ierr = PetscRandomGetValue(rctx,&v);CHKERRQ(ierr);
if (PetscRealPart(v) < .25 || i == j) {
for (k=0; k<bs; k++) {
for (l=0; l<bs; l++) {
ierr = PetscRandomGetValue(rctx,&v);CHKERRQ(ierr);
Ii = i*bs + k;
J = j*bs + l;
if (Ii == J) v += 10.;
ierr = MatSetValue(A,Ii,J,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr);
ierr = VecSetSizes(u,PETSC_DECIDE,n*bs);CHKERRQ(ierr);
ierr = VecSetFromOptions(u);CHKERRQ(ierr);
ierr = VecDuplicate(u,&b);CHKERRQ(ierr);
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
ierr = VecSet(u,1.0);CHKERRQ(ierr);
ierr = MatMult(A,u,b);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the linear solver and set various options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Check solution and clean up
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Check the error
*/
ierr = VecAXPY(x,-1.0,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
/*
Print convergence information. PetscPrintf() produces a single
print statement from all processes that share a communicator.
An alternative is PetscFPrintf(), which prints to a file.
*/
if (norm > .1) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of residual %g iterations %D bs %D\n",(double)norm,its,bs);CHKERRQ(ierr);
}
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);
/*
Always call PetscFinalize() before exiting a program. This routine
- finalizes the PETSc libraries as well as MPI
- provides summary and diagnostic information if certain runtime
options are chosen (e.g., -log_summary).
*/
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Vec x,b,u; /* approx solution, RHS, exact solution */
Mat A; /* linear system matrix */
KSP ksp; /* linear solver context */
PetscReal norm; /* norm of solution error */
PetscInt dim,i,j,Ii,J,Istart,Iend,n = 6,its,use_random;
PetscErrorCode ierr;
PetscScalar v,none = -1.0,sigma2,pfive = 0.5,*xa;
PetscRandom rctx;
PetscReal h2,sigma1 = 100.0;
PetscBool flg = PETSC_FALSE;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
#if !defined(PETSC_USE_COMPLEX)
SETERRQ(PETSC_COMM_WORLD,1,"This example requires complex numbers");
#endif
ierr = PetscOptionsGetReal(NULL,NULL,"-sigma1",&sigma1,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
dim = n*n;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compute the matrix and right-hand-side vector that define
the linear system, Ax = b.
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create parallel matrix, specifying only its global dimensions.
When using MatCreate(), the matrix format can be specified at
runtime. Also, the parallel partitioning of the matrix is
determined by PETSc at runtime.
*/
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,dim,dim);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/*
Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors. Determine which
rows of the matrix are locally owned.
*/
ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr);
/*
Set matrix elements in parallel.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Always specify global rows and columns of matrix entries.
*/
ierr = PetscOptionsGetBool(NULL,NULL,"-norandom",&flg,NULL);CHKERRQ(ierr);
if (flg) use_random = 0;
else use_random = 1;
if (use_random) {
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(rctx,0.0,PETSC_i);CHKERRQ(ierr);
} else {
sigma2 = 10.0*PETSC_i;
}
h2 = 1.0/((n+1)*(n+1));
for (Ii=Istart; Ii<Iend; Ii++) {
v = -1.0; i = Ii/n; j = Ii - i*n;
if (i>0) {
J = Ii-n; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);
}
if (i<n-1) {
J = Ii+n; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);
}
if (j>0) {
J = Ii-1; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);
}
if (j<n-1) {
J = Ii+1; ierr = MatSetValues(A,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);
}
if (use_random) {ierr = PetscRandomGetValue(rctx,&sigma2);CHKERRQ(ierr);}
v = 4.0 - sigma1*h2 + sigma2*h2;
ierr = MatSetValues(A,1,&Ii,1,&Ii,&v,ADD_VALUES);CHKERRQ(ierr);
}
if (use_random) {ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);}
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd()
Computations can be done while messages are in transition
by placing code between these two statements.
*/
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Create parallel vectors.
- When using VecCreate(), VecSetSizes() and VecSetFromOptions(),
we specify only the vector's global
dimension; the parallel partitioning is determined at runtime.
- Note: We form 1 vector from scratch and then duplicate as needed.
*/
ierr = VecCreate(PETSC_COMM_WORLD,&u);CHKERRQ(ierr);
ierr = VecSetSizes(u,PETSC_DECIDE,dim);CHKERRQ(ierr);
ierr = VecSetFromOptions(u);CHKERRQ(ierr);
ierr = VecDuplicate(u,&b);CHKERRQ(ierr);
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
/*
Set exact solution; then compute right-hand-side vector.
*/
if (use_random) {
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr);
ierr = VecSetRandom(u,rctx);CHKERRQ(ierr);
} else {
ierr = VecSet(u,pfive);CHKERRQ(ierr);
}
ierr = MatMult(A,u,b);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the linear solver and set various options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create linear solver context
*/
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
/*
Set operators. Here the matrix that defines the linear system
also serves as the preconditioning matrix.
*/
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
/*
Set runtime options, e.g.,
-ksp_type <type> -pc_type <type> -ksp_monitor -ksp_rtol <rtol>
*/
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the linear system
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = KSPSolve(ksp,b,x);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Check solution and clean up
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Print the first 3 entries of x; this demonstrates extraction of the
real and imaginary components of the complex vector, x.
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-print_x3",&flg,NULL);CHKERRQ(ierr);
if (flg) {
ierr = VecGetArray(x,&xa);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"The first three entries of x are:\n");CHKERRQ(ierr);
for (i=0; i<3; i++) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"x[%D] = %g + %g i\n",i,(double)PetscRealPart(xa[i]),(double)PetscImaginaryPart(xa[i]));CHKERRQ(ierr);
}
ierr = VecRestoreArray(x,&xa);CHKERRQ(ierr);
}
/*
Check the error
*/
ierr = VecAXPY(x,none,u);CHKERRQ(ierr);
ierr = VecNorm(x,NORM_2,&norm);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
if (norm < 1.e-12) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error < 1.e-12 iterations %D\n",its);CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm of error %g iterations %D\n",(double)norm,its);CHKERRQ(ierr);
}
/*
Free work space. All PETSc objects should be destroyed when they
are no longer needed.
*/
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
if (use_random) {ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);}
ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@
MatNullSpaceCreateRigidBody - create rigid body modes from coordinates
Collective on Vec
Input Argument:
. coords - block of coordinates of each node, must have block size set
Output Argument:
. sp - the null space
Level: advanced
.seealso: MatNullSpaceCreate()
@*/
PetscErrorCode MatNullSpaceCreateRigidBody(Vec coords,MatNullSpace *sp)
{
PetscErrorCode ierr;
const PetscScalar *x;
PetscScalar *v[6],dots[5];
Vec vec[6];
PetscInt n,N,dim,nmodes,i,j;
PetscFunctionBegin;
ierr = VecGetBlockSize(coords,&dim);CHKERRQ(ierr);
ierr = VecGetLocalSize(coords,&n);CHKERRQ(ierr);
ierr = VecGetSize(coords,&N);CHKERRQ(ierr);
n /= dim;
N /= dim;
switch (dim) {
case 1:
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)coords),PETSC_TRUE,0,NULL,sp);CHKERRQ(ierr);
break;
case 2:
case 3:
nmodes = (dim == 2) ? 3 : 6;
ierr = VecCreate(PetscObjectComm((PetscObject)coords),&vec[0]);CHKERRQ(ierr);
ierr = VecSetSizes(vec[0],dim*n,dim*N);CHKERRQ(ierr);
ierr = VecSetBlockSize(vec[0],dim);CHKERRQ(ierr);
ierr = VecSetUp(vec[0]);CHKERRQ(ierr);
for (i=1; i<nmodes; i++) {ierr = VecDuplicate(vec[0],&vec[i]);CHKERRQ(ierr);}
for (i=0; i<nmodes; i++) {ierr = VecGetArray(vec[i],&v[i]);CHKERRQ(ierr);}
ierr = VecGetArrayRead(coords,&x);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (dim == 2) {
v[0][i*2+0] = 1./N;
v[0][i*2+1] = 0.;
v[1][i*2+0] = 0.;
v[1][i*2+1] = 1./N;
/* Rotations */
v[2][i*2+0] = -x[i*2+1];
v[2][i*2+1] = x[i*2+0];
} else {
v[0][i*3+0] = 1./N;
v[0][i*3+1] = 0.;
v[0][i*3+2] = 0.;
v[1][i*3+0] = 0.;
v[1][i*3+1] = 1./N;
v[1][i*3+2] = 0.;
v[2][i*3+0] = 0.;
v[2][i*3+1] = 0.;
v[2][i*3+2] = 1./N;
v[3][i*3+0] = x[i*3+1];
v[3][i*3+1] = -x[i*3+0];
v[3][i*3+2] = 0.;
v[4][i*3+0] = 0.;
v[4][i*3+1] = -x[i*3+2];
v[4][i*3+2] = x[i*3+1];
v[5][i*3+0] = x[i*3+2];
v[5][i*3+1] = 0.;
v[5][i*3+2] = -x[i*3+0];
}
}
for (i=0; i<nmodes; i++) {ierr = VecRestoreArray(vec[i],&v[i]);CHKERRQ(ierr);}
ierr = VecRestoreArrayRead(coords,&x);CHKERRQ(ierr);
for (i=dim; i<nmodes; i++) {
/* Orthonormalize vec[i] against vec[0:i-1] */
ierr = VecMDot(vec[i],i,vec,dots);CHKERRQ(ierr);
for (j=0; j<i; j++) dots[j] *= -1.;
ierr = VecMAXPY(vec[i],i,dots,vec);CHKERRQ(ierr);
ierr = VecNormalize(vec[i],NULL);CHKERRQ(ierr);
}
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)coords),PETSC_FALSE,nmodes,vec,sp);CHKERRQ(ierr);
for (i=0; i<nmodes; i++) {ierr = VecDestroy(&vec[i]);CHKERRQ(ierr);}
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da,*subda;
PetscInt i,dim=3;
PetscMPIInt size,rank;
Vec v;
Vec slvec,sgvec;
IS *ois,*iis;
VecScatter oscata;
VecScatter *iscat,*oscat,*gscat;
DMDALocalInfo info;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-dim",&dim,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,-4,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMCreateDomainDecomposition(da,NULL,NULL,&iis,&ois,&subda);CHKERRQ(ierr);
ierr = DMCreateDomainDecompositionScatters(da,1,subda,&iscat,&oscat,&gscat);CHKERRQ(ierr);
{
DMDALocalInfo subinfo;
MatStencil lower,upper;
IS patchis,subpatchis;
Vec smallvec;
Vec largevec;
VecScatter patchscat;
ierr = DMDAGetLocalInfo(subda[0],&subinfo);CHKERRQ(ierr);
lower.i = info.xs;
lower.j = info.ys;
lower.k = info.zs;
upper.i = info.xs+info.xm;
upper.j = info.ys+info.ym;
upper.k = info.zs+info.zm;
/* test the patch IS as a thing to scatter to/from */
ierr = DMDACreatePatchIS(da,&lower,&upper,&patchis);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&smallvec);CHKERRQ(ierr);
ierr = VecSetSizes(smallvec,info.dof*(upper.i - lower.i)*(upper.j - lower.j)*(upper.k - lower.k),PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(smallvec);CHKERRQ(ierr);
ierr = VecScatterCreate(smallvec,NULL,largevec,patchis,&patchscat);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],smallvec);CHKERRQ(ierr);
ierr = VecSet(largevec,0);CHKERRQ(ierr);
ierr = VecScatterBegin(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = ISView(patchis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecScatterView(patchscat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecView(smallvec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecView(largevec,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&smallvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = ISDestroy(&patchis);CHKERRQ(ierr);
ierr = VecScatterDestroy(&patchscat);CHKERRQ(ierr);
}
/* view the various parts */
{
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMView(subda[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&v);CHKERRQ(ierr);
/* test filling outer between the big DM and the small ones with the IS scatter*/
ierr = VecScatterCreate(v,ois[0],sgvec,NULL,&oscata);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],sgvec);CHKERRQ(ierr);
ierr = VecScatterBegin(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* test the local-to-local scatter */
/* fill up the local subdomain and then add them together */
ierr = FillLocalSubdomain(da,v);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test ghost scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test overlap scattering backwards */
ierr = DMLocalToGlobalBegin(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test interior scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test matrix allocation */
for (i = 0; i < size; i++) {
if (i == rank) {
Mat m;
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMSetMatType(subda[0],MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(subda[0],&m);CHKERRQ(ierr);
ierr = MatView(m,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&m);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMRestoreLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&v);CHKERRQ(ierr);
}
ierr = DMDestroy(&subda[0]);CHKERRQ(ierr);
ierr = ISDestroy(&ois[0]);CHKERRQ(ierr);
ierr = ISDestroy(&iis[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&iscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&gscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscata);CHKERRQ(ierr);
ierr = PetscFree(iscat);CHKERRQ(ierr);
ierr = PetscFree(oscat);CHKERRQ(ierr);
ierr = PetscFree(gscat);CHKERRQ(ierr);
ierr = PetscFree(oscata);CHKERRQ(ierr);
ierr = PetscFree(subda);CHKERRQ(ierr);
ierr = PetscFree(ois);CHKERRQ(ierr);
ierr = PetscFree(iis);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PETSC_STATIC_INLINE PetscErrorCode DMInterpolate_Hex_Private(DMInterpolationInfo ctx, DM dm, Vec xLocal, Vec v)
{
DM dmCoord;
SNES snes;
KSP ksp;
PC pc;
Vec coordsLocal, r, ref, real;
Mat J;
PetscScalar *a, *coords;
PetscInt p;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMGetCoordinatesLocal(dm, &coordsLocal);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(dm, &dmCoord);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_SELF, &snes);CHKERRQ(ierr);
ierr = SNESSetOptionsPrefix(snes, "hex_interp_");CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF, &r);CHKERRQ(ierr);
ierr = VecSetSizes(r, 3, 3);CHKERRQ(ierr);
ierr = VecSetType(r,dm->vectype);CHKERRQ(ierr);
ierr = VecDuplicate(r, &ref);CHKERRQ(ierr);
ierr = VecDuplicate(r, &real);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_SELF, &J);CHKERRQ(ierr);
ierr = MatSetSizes(J, 3, 3, 3, 3);CHKERRQ(ierr);
ierr = MatSetType(J, MATSEQDENSE);CHKERRQ(ierr);
ierr = MatSetUp(J);CHKERRQ(ierr);
ierr = SNESSetFunction(snes, r, HexMap_Private, NULL);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, J, J, HexJacobian_Private, NULL);CHKERRQ(ierr);
ierr = SNESGetKSP(snes, &ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp, &pc);CHKERRQ(ierr);
ierr = PCSetType(pc, PCLU);CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes);CHKERRQ(ierr);
ierr = VecGetArray(ctx->coords, &coords);CHKERRQ(ierr);
ierr = VecGetArray(v, &a);CHKERRQ(ierr);
for (p = 0; p < ctx->n; ++p) {
PetscScalar *x = NULL, *vertices = NULL;
PetscScalar *xi;
PetscReal xir[3];
PetscInt c = ctx->cells[p], comp, coordSize, xSize;
/* Can make this do all points at once */
ierr = DMPlexVecGetClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);CHKERRQ(ierr);
if (8*3 != coordSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", coordSize, 8*3);
ierr = DMPlexVecGetClosure(dm, NULL, xLocal, c, &xSize, &x);CHKERRQ(ierr);
if (8*ctx->dof != xSize) SETERRQ2(ctx->comm, PETSC_ERR_ARG_SIZ, "Invalid closure size %d should be %d", xSize, 8*ctx->dof);
ierr = SNESSetFunction(snes, NULL, NULL, (void*) vertices);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes, NULL, NULL, NULL, (void*) vertices);CHKERRQ(ierr);
ierr = VecGetArray(real, &xi);CHKERRQ(ierr);
xi[0] = coords[p*ctx->dim+0];
xi[1] = coords[p*ctx->dim+1];
xi[2] = coords[p*ctx->dim+2];
ierr = VecRestoreArray(real, &xi);CHKERRQ(ierr);
ierr = SNESSolve(snes, real, ref);CHKERRQ(ierr);
ierr = VecGetArray(ref, &xi);CHKERRQ(ierr);
xir[0] = PetscRealPart(xi[0]);
xir[1] = PetscRealPart(xi[1]);
xir[2] = PetscRealPart(xi[2]);
for (comp = 0; comp < ctx->dof; ++comp) {
a[p*ctx->dof+comp] =
x[0*ctx->dof+comp]*(1-xir[0])*(1-xir[1])*(1-xir[2]) +
x[3*ctx->dof+comp]* xir[0]*(1-xir[1])*(1-xir[2]) +
x[2*ctx->dof+comp]* xir[0]* xir[1]*(1-xir[2]) +
x[1*ctx->dof+comp]*(1-xir[0])* xir[1]*(1-xir[2]) +
x[4*ctx->dof+comp]*(1-xir[0])*(1-xir[1])* xir[2] +
x[5*ctx->dof+comp]* xir[0]*(1-xir[1])* xir[2] +
x[6*ctx->dof+comp]* xir[0]* xir[1]* xir[2] +
x[7*ctx->dof+comp]*(1-xir[0])* xir[1]* xir[2];
}
ierr = VecRestoreArray(ref, &xi);CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dmCoord, NULL, coordsLocal, c, &coordSize, &vertices);CHKERRQ(ierr);
ierr = DMPlexVecRestoreClosure(dm, NULL, xLocal, c, &xSize, &x);CHKERRQ(ierr);
}
ierr = VecRestoreArray(v, &a);CHKERRQ(ierr);
ierr = VecRestoreArray(ctx->coords, &coords);CHKERRQ(ierr);
ierr = SNESDestroy(&snes);CHKERRQ(ierr);
ierr = VecDestroy(&r);CHKERRQ(ierr);
ierr = VecDestroy(&ref);CHKERRQ(ierr);
ierr = VecDestroy(&real);CHKERRQ(ierr);
ierr = MatDestroy(&J);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatFactorNumeric_PaStiX(Mat F,Mat A,const MatFactorInfo *info)
{
Mat_Pastix *lu =(Mat_Pastix*)(F)->spptr;
Mat *tseq;
PetscErrorCode ierr = 0;
PetscInt icntl;
PetscInt M=A->rmap->N;
PetscBool valOnly,flg, isSym;
Mat F_diag;
IS is_iden;
Vec b;
IS isrow;
PetscBool isSeqAIJ,isSeqSBAIJ,isMPIAIJ;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isSeqAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&isMPIAIJ);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQSBAIJ,&isSeqSBAIJ);CHKERRQ(ierr);
if (lu->matstruc == DIFFERENT_NONZERO_PATTERN){
(F)->ops->solve = MatSolve_PaStiX;
/* Initialize a PASTIX instance */
ierr = MPI_Comm_dup(((PetscObject)A)->comm,&(lu->pastix_comm));CHKERRQ(ierr);
ierr = MPI_Comm_rank(lu->pastix_comm, &lu->commRank); CHKERRQ(ierr);
ierr = MPI_Comm_size(lu->pastix_comm, &lu->commSize); CHKERRQ(ierr);
/* Set pastix options */
lu->iparm[IPARM_MODIFY_PARAMETER] = API_NO;
lu->iparm[IPARM_START_TASK] = API_TASK_INIT;
lu->iparm[IPARM_END_TASK] = API_TASK_INIT;
lu->rhsnbr = 1;
/* Call to set default pastix options */
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
ierr = PetscOptionsBegin(((PetscObject)A)->comm,((PetscObject)A)->prefix,"PaStiX Options","Mat");CHKERRQ(ierr);
icntl=-1;
lu->iparm[IPARM_VERBOSE] = API_VERBOSE_NOT;
ierr = PetscOptionsInt("-mat_pastix_verbose","iparm[IPARM_VERBOSE] : level of printing (0 to 2)","None",lu->iparm[IPARM_VERBOSE],&icntl,&flg);CHKERRQ(ierr);
if ((flg && icntl >= 0) || PetscLogPrintInfo) {
lu->iparm[IPARM_VERBOSE] = icntl;
}
icntl=-1;
ierr = PetscOptionsInt("-mat_pastix_threadnbr","iparm[IPARM_THREAD_NBR] : Number of thread by MPI node","None",lu->iparm[IPARM_THREAD_NBR],&icntl,PETSC_NULL);CHKERRQ(ierr);
if ((flg && icntl > 0)) {
lu->iparm[IPARM_THREAD_NBR] = icntl;
}
PetscOptionsEnd();
valOnly = PETSC_FALSE;
} else {
if (isSeqAIJ || isMPIAIJ) {
ierr = PetscFree(lu->colptr);CHKERRQ(ierr);
ierr = PetscFree(lu->row);CHKERRQ(ierr);
ierr = PetscFree(lu->val);CHKERRQ(ierr);
valOnly = PETSC_FALSE;
} else valOnly = PETSC_TRUE;
}
lu->iparm[IPARM_MATRIX_VERIFICATION] = API_YES;
/* convert mpi A to seq mat A */
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&tseq);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
ierr = MatConvertToCSC(*tseq,valOnly, &lu->n, &lu->colptr, &lu->row, &lu->val);CHKERRQ(ierr);
ierr = MatIsSymmetric(*tseq,0.0,&isSym);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&tseq);CHKERRQ(ierr);
if (!lu->perm) {
ierr = PetscMalloc((lu->n)*sizeof(PetscInt) ,&(lu->perm));CHKERRQ(ierr);
ierr = PetscMalloc((lu->n)*sizeof(PetscInt) ,&(lu->invp));CHKERRQ(ierr);
}
if (isSym) {
/* On symmetric matrix, LLT */
lu->iparm[IPARM_SYM] = API_SYM_YES;
lu->iparm[IPARM_FACTORIZATION] = API_FACT_LDLT;
} else {
/* On unsymmetric matrix, LU */
lu->iparm[IPARM_SYM] = API_SYM_NO;
lu->iparm[IPARM_FACTORIZATION] = API_FACT_LU;
}
/*----------------*/
if (lu->matstruc == DIFFERENT_NONZERO_PATTERN){
if (!(isSeqAIJ || isSeqSBAIJ)) {
/* PaStiX only supports centralized rhs. Create scatter scat_rhs for repeated use in MatSolve() */
ierr = VecCreateSeq(PETSC_COMM_SELF,A->cmap->N,&lu->b_seq);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,A->cmap->N,0,1,&is_iden);CHKERRQ(ierr);
ierr = VecCreate(((PetscObject)A)->comm,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,A->rmap->n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
ierr = VecScatterCreate(b,is_iden,lu->b_seq,is_iden,&lu->scat_rhs);CHKERRQ(ierr);
ierr = VecScatterCreate(lu->b_seq,is_iden,b,is_iden,&lu->scat_sol);CHKERRQ(ierr);
ierr = ISDestroy(&is_iden);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
}
lu->iparm[IPARM_START_TASK] = API_TASK_ORDERING;
lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]);
} else {
lu->iparm[IPARM_START_TASK] = API_TASK_NUMFACT;
lu->iparm[IPARM_END_TASK] = API_TASK_NUMFACT;
PASTIX_CALL(&(lu->pastix_data),
lu->pastix_comm,
lu->n,
lu->colptr,
lu->row,
(PastixScalar*)lu->val,
lu->perm,
lu->invp,
(PastixScalar*)lu->rhs,
lu->rhsnbr,
lu->iparm,
lu->dparm);
if (lu->iparm[IPARM_ERROR_NUMBER] < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Error reported by PaStiX in analysis phase: iparm(IPARM_ERROR_NUMBER)=%d\n",lu->iparm[IPARM_ERROR_NUMBER]);
}
if (lu->commSize > 1){
if ((F)->factortype == MAT_FACTOR_LU){
F_diag = ((Mat_MPIAIJ *)(F)->data)->A;
} else {
F_diag = ((Mat_MPISBAIJ *)(F)->data)->A;
}
F_diag->assembled = PETSC_TRUE;
}
(F)->assembled = PETSC_TRUE;
lu->matstruc = SAME_NONZERO_PATTERN;
lu->CleanUpPastix = PETSC_TRUE;
PetscFunctionReturn(0);
}
PetscErrorCode DMInterpolationSetUp(DMInterpolationInfo ctx, DM dm, PetscBool redundantPoints)
{
MPI_Comm comm = ctx->comm;
PetscScalar *a;
PetscInt p, q, i;
PetscMPIInt rank, size;
PetscErrorCode ierr;
Vec pointVec;
IS cellIS;
PetscLayout layout;
PetscReal *globalPoints;
PetscScalar *globalPointsScalar;
const PetscInt *ranges;
PetscMPIInt *counts, *displs;
const PetscInt *foundCells;
PetscMPIInt *foundProcs, *globalProcs;
PetscInt n, N;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
if (ctx->dim < 0) SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set");
/* Locate points */
n = ctx->nInput;
if (!redundantPoints) {
ierr = PetscLayoutCreate(comm, &layout);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(layout, n);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(layout);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(layout, &N);CHKERRQ(ierr);
/* Communicate all points to all processes */
ierr = PetscMalloc3(N*ctx->dim,&globalPoints,size,&counts,size,&displs);CHKERRQ(ierr);
ierr = PetscLayoutGetRanges(layout, &ranges);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
counts[p] = (ranges[p+1] - ranges[p])*ctx->dim;
displs[p] = ranges[p]*ctx->dim;
}
ierr = MPI_Allgatherv(ctx->points, n*ctx->dim, MPIU_REAL, globalPoints, counts, displs, MPIU_REAL, comm);CHKERRQ(ierr);
} else {
N = n;
globalPoints = ctx->points;
counts = displs = NULL;
layout = NULL;
}
#if 0
ierr = PetscMalloc3(N,&foundCells,N,&foundProcs,N,&globalProcs);CHKERRQ(ierr);
/* foundCells[p] = m->locatePoint(&globalPoints[p*ctx->dim]); */
#else
#if defined(PETSC_USE_COMPLEX)
ierr = PetscMalloc1(N,&globalPointsScalar);CHKERRQ(ierr);
for (i=0; i<N; i++) globalPointsScalar[i] = globalPoints[i];
#else
globalPointsScalar = globalPoints;
#endif
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, ctx->dim, N*ctx->dim, globalPointsScalar, &pointVec);CHKERRQ(ierr);
ierr = PetscMalloc2(N,&foundProcs,N,&globalProcs);CHKERRQ(ierr);
ierr = DMLocatePoints(dm, pointVec, &cellIS);CHKERRQ(ierr);
ierr = ISGetIndices(cellIS, &foundCells);CHKERRQ(ierr);
#endif
for (p = 0; p < N; ++p) {
if (foundCells[p] >= 0) foundProcs[p] = rank;
else foundProcs[p] = size;
}
/* Let the lowest rank process own each point */
ierr = MPI_Allreduce(foundProcs, globalProcs, N, MPI_INT, MPI_MIN, comm);CHKERRQ(ierr);
ctx->n = 0;
for (p = 0; p < N; ++p) {
if (globalProcs[p] == size) SETERRQ4(comm, PETSC_ERR_PLIB, "Point %d: %g %g %g not located in mesh", p, globalPoints[p*ctx->dim+0], ctx->dim > 1 ? globalPoints[p*ctx->dim+1] : 0.0, ctx->dim > 2 ? globalPoints[p*ctx->dim+2] : 0.0);
else if (globalProcs[p] == rank) ctx->n++;
}
/* Create coordinates vector and array of owned cells */
ierr = PetscMalloc1(ctx->n, &ctx->cells);CHKERRQ(ierr);
ierr = VecCreate(comm, &ctx->coords);CHKERRQ(ierr);
ierr = VecSetSizes(ctx->coords, ctx->n*ctx->dim, PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetBlockSize(ctx->coords, ctx->dim);CHKERRQ(ierr);
ierr = VecSetType(ctx->coords,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(ctx->coords, &a);CHKERRQ(ierr);
for (p = 0, q = 0, i = 0; p < N; ++p) {
if (globalProcs[p] == rank) {
PetscInt d;
for (d = 0; d < ctx->dim; ++d, ++i) a[i] = globalPoints[p*ctx->dim+d];
ctx->cells[q++] = foundCells[p];
}
}
ierr = VecRestoreArray(ctx->coords, &a);CHKERRQ(ierr);
#if 0
ierr = PetscFree3(foundCells,foundProcs,globalProcs);CHKERRQ(ierr);
#else
ierr = PetscFree2(foundProcs,globalProcs);CHKERRQ(ierr);
ierr = ISRestoreIndices(cellIS, &foundCells);CHKERRQ(ierr);
ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
ierr = VecDestroy(&pointVec);CHKERRQ(ierr);
#endif
if ((void*)globalPointsScalar != (void*)globalPoints) {ierr = PetscFree(globalPointsScalar);CHKERRQ(ierr);}
if (!redundantPoints) {ierr = PetscFree3(globalPoints,counts,displs);CHKERRQ(ierr);}
ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A; /* linear system matrix */
PetscErrorCode ierr;
PetscMPIInt rank=0;
PetscBool flg;
PetscViewer fd; /* viewer */
PetscViewer log;
char file[PETSC_MAX_PATH_LEN];
char logfile[PETSC_MAX_PATH_LEN];
char lockfile[PETSC_MAX_PATH_LEN], tmpstr[PETSC_MAX_PATH_LEN], dirname[PETSC_MAX_PATH_LEN], matrix[PETSC_MAX_PATH_LEN];
char hash[20];
PetscLogDouble solveTime,endTime,startTime;
PetscInt its;
PetscReal norm;
KSP ksp; // Linear solver context
Vec b,x,u; // RHS, solution, vector for norm calculation
PetscScalar one = 1.0;
PetscInt m, n, i;
FILE *lock;
/*
if (rank == 0) {
printf("Command line arguments:\n");
for (i=0; i < argc; i++)
printf("%d: %s\n", i, args[i]);
}
// Save args
int argcount = argc;
char **argv = (char**) malloc (argc*sizeof(char*));
for (i=0; i < argc; i++) {
argv[i] = (char*) malloc(strlen(args[i]) + 1);
strcpy(argv[i],args[i]);
}
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
*/
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-hash",hash,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) {
strcpy(hash,"nohash");
}
ierr = PetscOptionsGetString(PETSC_NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) {
PetscPrintf(PETSC_COMM_WORLD,"Must indicate matrix file with the -f option");
}
/* Create lock file */
if (rank == 0) {
for (i = strlen(file); i> 0; i--) if (file[i] == '.') break;
strncpy(tmpstr, file, i-1);
for (i = strlen(tmpstr); i> 0; i--) if (file[i] == '/') break;
strncpy(dirname, tmpstr, i);
dirname[i] = '\0';
sprintf(lockfile,"%s/../timing/.%s.%s", dirname, basename(tmpstr), hash);
sprintf(logfile,"%s/../timing/%s.%s.log", dirname, basename(tmpstr), hash);
lock = fopen(lockfile, "w");
fprintf(lock, "%s\n", file);
fclose(lock);
}
/* Read file */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
// Create matrix
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetType(A,MATMPIAIJ); CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
// Load matrix from file
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
ierr = MatGetSize(A, &m, &n); CHKERRQ(ierr);
// Assemble matrix
//ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
//ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
// Create RHS vector
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,PETSC_DECIDE,n); CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
ierr = VecSet(b,one); CHKERRQ(ierr);
//ierr = VecLoad(b,fd);CHKERRQ(ierr);
// Create vectors x and u
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&u);CHKERRQ(ierr);
// Create KSP
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp); CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp,PETSC_FALSE);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr);
// Setup KSP
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
ierr = KSPSetUpOnBlocks(ksp);CHKERRQ(ierr);
// Get start time
ierr = PetscTime(&startTime);CHKERRQ(ierr);
// Get KSP and PC type
KSPType kt;
ierr = KSPGetType(ksp,&kt);
PC pc;
ierr = KSPGetPC(ksp,&pc);
PCType pt;
ierr = PCGetType(pc,&pt);
// Print method info
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, logfile, &log); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(log, "Hash: %s\n", hash);
ierr = PetscViewerASCIIPrintf(log, "%s | %s",kt,pt);CHKERRQ(ierr);
// Make sure the program doesn't crash
// while trying to solve the system
PetscPushErrorHandler(PetscIgnoreErrorHandler,NULL);
ierr = KSPSolve(ksp,b,x);
PetscPopErrorHandler();
// Check if anything went wrong
if(ierr == 0 || ierr == -1){
// If no error occurred or stopped by MyKSPMonitor,
// compute normal and stuff
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = MatMult(A,x,u);CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscTime(&endTime);CHKERRQ(ierr);
// Compute solve time
solveTime = endTime - startTime;
// Check if KSP converged
KSPConvergedReason reason;
KSPGetConvergedReason(ksp,&reason);
// Print convergence code, solve time, preconditioned norm, iterations
ierr = PetscViewerASCIIPrintf(log, " | %D | %e | %g | %D\n",reason,solveTime,norm,its);CHKERRQ(ierr);
ierr = KSPView(ksp,log);
ierr = PCView(pc,log);
ierr = PetscLogView(log);
}
else{
// Disaster happened, bail out
if (rank == 0) remove(lockfile);
PetscFinalize();
return 0;
}
// Again, destroy KSP and vector
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
if (rank == 0) remove(lockfile);
PetscFinalize();
return 0;
}
/* Two triangles separated by a zero-volume cell with 6 vertices
5--16--8
/ | | \
11 | | 12
/ | | \
3 0 10 2 14 1 6
\ | | /
9 | | 13
\ | | /
4--15--7
*/
PetscErrorCode CreateSimplexHybrid_2D(MPI_Comm comm, DM dm)
{
Vec coordinates;
PetscSection coordSection;
PetscScalar *coords;
PetscInt numVertices = 0, numEdges = 0, numCells = 0, cMax = PETSC_DETERMINE, fMax = PETSC_DETERMINE;
PetscInt firstVertex, firstEdge, coordSize;
PetscInt v, e;
PetscMPIInt rank;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
if (!rank) {
numVertices = 3 + 3;
numEdges = 6 + 2;
numCells = 3;
cMax = 2;
fMax = 15;
}
firstVertex = numCells;
firstEdge = numCells + numVertices;
ierr = DMPlexSetChart(dm, 0, numCells+numEdges+numVertices);CHKERRQ(ierr);
if (numCells) {
ierr = DMPlexSetConeSize(dm, 0, 3);CHKERRQ(ierr);
ierr = DMPlexSetConeSize(dm, 1, 3);CHKERRQ(ierr);
ierr = DMPlexSetConeSize(dm, 2, 4);CHKERRQ(ierr);
}
for(e = firstEdge; e < firstEdge+numEdges; ++e) {
ierr = DMPlexSetConeSize(dm, e, 2);CHKERRQ(ierr);
}
ierr = DMSetUp(dm);CHKERRQ(ierr); /* Allocate space for cones */
ierr = DMPlexSetHybridBounds(dm, cMax, PETSC_DETERMINE, fMax, PETSC_DETERMINE);CHKERRQ(ierr); /* Indicate a hybrid mesh */
/* Build cells */
if (numCells > 0) {
const PetscInt cone[3] = {9, 10, 11};
const PetscInt ornt[3] = {0, 0, 0};
const PetscInt cell = 0;
ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
ierr = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
}
if (numCells > 1) {
const PetscInt cone[3] = {12, 14, 13};
const PetscInt ornt[3] = { 0, -2, 0};
const PetscInt cell = 1;
ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
ierr = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
}
if (numCells > 2) {
const PetscInt cone[4] = {10, 14, 15, 16};
const PetscInt ornt[4] = { 0, 0, 0, 0};
const PetscInt cell = 2;
ierr = DMPlexSetCone(dm, cell, cone);CHKERRQ(ierr);
ierr = DMPlexSetConeOrientation(dm, cell, ornt);CHKERRQ(ierr);
}
/* Build edges*/
if (numEdges > 0) {
const PetscInt cone[2] = {3, 4};
const PetscInt edge = 9;
ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr);
}
if (numEdges > 1) {
const PetscInt cone[2] = {4, 5};
const PetscInt edge = 10;
ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
}
if (numEdges > 2) {
const PetscInt cone[2] = {5, 3};
const PetscInt edge = 11;
ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr);
}
if (numEdges > 3) {
const PetscInt cone[2] = {6, 8};
const PetscInt edge = 12;
ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr);
}
if (numEdges > 4) {
const PetscInt cone[2] = {7, 6};
const PetscInt edge = 13;
ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(dm, "marker", edge, 1);CHKERRQ(ierr);
}
if (numEdges > 5) {
const PetscInt cone[2] = {7, 8};
const PetscInt edge = 14;
ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
}
if (numEdges > 6) {
const PetscInt cone[2] = {4, 7};
const PetscInt edge = 15;
ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
}
if (numEdges > 7) {
const PetscInt cone[2] = {5, 8};
const PetscInt edge = 16;
ierr = DMPlexSetCone(dm, edge, cone);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(dm);CHKERRQ(ierr);
ierr = DMPlexStratify(dm);CHKERRQ(ierr);
/* Build coordinates */
ierr = DMPlexGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices);CHKERRQ(ierr);
for(v = firstVertex; v < firstVertex+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, 2);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(((PetscObject) dm)->comm, &coordinates);CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetFromOptions(coordinates);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (numVertices) {
coords[0] = -0.5; coords[1] = 0.5;
coords[2] = -0.2; coords[3] = 0.0;
coords[4] = -0.2; coords[5] = 1.0;
coords[6] = 0.5; coords[7] = 0.5;
coords[8] = 0.2; coords[9] = 0.0;
coords[10] = 0.2; coords[11] = 1.0;
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCSetupFETIDPMatContext(FETIDPMat_ctx fetidpmat_ctx )
{
PetscErrorCode ierr;
PC_IS *pcis=(PC_IS*)fetidpmat_ctx->pc->data;
PC_BDDC *pcbddc=(PC_BDDC*)fetidpmat_ctx->pc->data;
PCBDDCGraph mat_graph=pcbddc->mat_graph;
Mat_IS *matis = (Mat_IS*)fetidpmat_ctx->pc->pmat->data;
MPI_Comm comm;
Mat ScalingMat;
Vec lambda_global;
IS IS_l2g_lambda;
IS subset,subset_mult,subset_n;
PetscBool skip_node,fully_redundant;
PetscInt i,j,k,s,n_boundary_dofs,n_global_lambda,n_vertices,partial_sum;
PetscInt cum,n_local_lambda,n_lambda_for_dof,dual_size,n_neg_values,n_pos_values;
PetscMPIInt rank,size,buf_size,neigh;
PetscScalar scalar_value;
PetscInt *vertex_indices;
PetscInt *dual_dofs_boundary_indices,*aux_local_numbering_1;
const PetscInt *aux_global_numbering;
PetscInt *aux_sums,*cols_B_delta,*l2g_indices;
PetscScalar *array,*scaling_factors,*vals_B_delta;
PetscInt *aux_local_numbering_2;
/* For communication of scaling factors */
PetscInt *ptrs_buffer,neigh_position;
PetscScalar **all_factors,*send_buffer,*recv_buffer;
MPI_Request *send_reqs,*recv_reqs;
/* tests */
Vec test_vec;
PetscBool test_fetidp;
PetscViewer viewer;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)(fetidpmat_ctx->pc),&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
/* Default type of lagrange multipliers is non-redundant */
fully_redundant = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-fetidp_fullyredundant",&fully_redundant,NULL);CHKERRQ(ierr);
/* Evaluate local and global number of lagrange multipliers */
ierr = VecSet(pcis->vec1_N,0.0);CHKERRQ(ierr);
n_local_lambda = 0;
partial_sum = 0;
n_boundary_dofs = 0;
s = 0;
/* Get Vertices used to define the BDDC */
n_vertices = pcbddc->n_vertices;
vertex_indices = pcbddc->local_primal_ref_node;
dual_size = pcis->n_B-n_vertices;
ierr = PetscMalloc1(dual_size,&dual_dofs_boundary_indices);CHKERRQ(ierr);
ierr = PetscMalloc1(dual_size,&aux_local_numbering_1);CHKERRQ(ierr);
ierr = PetscMalloc1(dual_size,&aux_local_numbering_2);CHKERRQ(ierr);
ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr);
for (i=0;i<pcis->n;i++){
j = mat_graph->count[i]; /* RECALL: mat_graph->count[i] does not count myself */
if ( j > 0 ) {
n_boundary_dofs++;
}
skip_node = PETSC_FALSE;
if ( s < n_vertices && vertex_indices[s]==i) { /* it works for a sorted set of vertices */
skip_node = PETSC_TRUE;
s++;
}
if (j < 1) {
skip_node = PETSC_TRUE;
}
if ( !skip_node ) {
if (fully_redundant) {
/* fully redundant set of lagrange multipliers */
n_lambda_for_dof = (j*(j+1))/2;
} else {
n_lambda_for_dof = j;
}
n_local_lambda += j;
/* needed to evaluate global number of lagrange multipliers */
array[i]=(1.0*n_lambda_for_dof)/(j+1.0); /* already scaled for the next global sum */
/* store some data needed */
dual_dofs_boundary_indices[partial_sum] = n_boundary_dofs-1;
aux_local_numbering_1[partial_sum] = i;
aux_local_numbering_2[partial_sum] = n_lambda_for_dof;
partial_sum++;
}
}
ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr);
ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecSum(pcis->vec1_global,&scalar_value);CHKERRQ(ierr);
fetidpmat_ctx->n_lambda = (PetscInt)PetscRealPart(scalar_value);
/* compute global ordering of lagrange multipliers and associate l2g map */
ierr = ISCreateGeneral(comm,partial_sum,aux_local_numbering_1,PETSC_COPY_VALUES,&subset_n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApplyIS(pcis->mapping,subset_n,&subset);CHKERRQ(ierr);
ierr = ISDestroy(&subset_n);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,partial_sum,aux_local_numbering_2,PETSC_OWN_POINTER,&subset_mult);CHKERRQ(ierr);
ierr = PCBDDCSubsetNumbering(subset,subset_mult,&i,&subset_n);CHKERRQ(ierr);
ierr = ISDestroy(&subset);CHKERRQ(ierr);
if (i != fetidpmat_ctx->n_lambda) {
SETERRQ3(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"Error in %s: global number of multipliers mismatch! (%d!=%d)\n",__FUNCT__,fetidpmat_ctx->n_lambda,i);
}
/* init data for scaling factors exchange */
partial_sum = 0;
j = 0;
ierr = PetscMalloc1(pcis->n_neigh,&ptrs_buffer);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n_neigh-1,&send_reqs);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n_neigh-1,&recv_reqs);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n,&all_factors);CHKERRQ(ierr);
ptrs_buffer[0]=0;
for (i=1;i<pcis->n_neigh;i++) {
partial_sum += pcis->n_shared[i];
ptrs_buffer[i] = ptrs_buffer[i-1]+pcis->n_shared[i];
}
ierr = PetscMalloc1(partial_sum,&send_buffer);CHKERRQ(ierr);
ierr = PetscMalloc1(partial_sum,&recv_buffer);CHKERRQ(ierr);
ierr = PetscMalloc1(partial_sum,&all_factors[0]);CHKERRQ(ierr);
for (i=0;i<pcis->n-1;i++) {
j = mat_graph->count[i];
all_factors[i+1]=all_factors[i]+j;
}
/* scatter B scaling to N vec */
ierr = VecScatterBegin(pcis->N_to_B,pcis->D,pcis->vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->N_to_B,pcis->D,pcis->vec1_N,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* communications */
ierr = VecGetArrayRead(pcis->vec1_N,(const PetscScalar**)&array);CHKERRQ(ierr);
for (i=1;i<pcis->n_neigh;i++) {
for (j=0;j<pcis->n_shared[i];j++) {
send_buffer[ptrs_buffer[i-1]+j]=array[pcis->shared[i][j]];
}
ierr = PetscMPIIntCast(ptrs_buffer[i]-ptrs_buffer[i-1],&buf_size);CHKERRQ(ierr);
ierr = PetscMPIIntCast(pcis->neigh[i],&neigh);CHKERRQ(ierr);
ierr = MPI_Isend(&send_buffer[ptrs_buffer[i-1]],buf_size,MPIU_SCALAR,neigh,0,comm,&send_reqs[i-1]);CHKERRQ(ierr);
ierr = MPI_Irecv(&recv_buffer[ptrs_buffer[i-1]],buf_size,MPIU_SCALAR,neigh,0,comm,&recv_reqs[i-1]);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(pcis->vec1_N,(const PetscScalar**)&array);CHKERRQ(ierr);
ierr = MPI_Waitall((pcis->n_neigh-1),recv_reqs,MPI_STATUSES_IGNORE);CHKERRQ(ierr);
/* put values in correct places */
for (i=1;i<pcis->n_neigh;i++) {
for (j=0;j<pcis->n_shared[i];j++) {
k = pcis->shared[i][j];
neigh_position = 0;
while(mat_graph->neighbours_set[k][neigh_position] != pcis->neigh[i]) {neigh_position++;}
all_factors[k][neigh_position]=recv_buffer[ptrs_buffer[i-1]+j];
}
}
ierr = MPI_Waitall((pcis->n_neigh-1),send_reqs,MPI_STATUSES_IGNORE);CHKERRQ(ierr);
ierr = PetscFree(send_reqs);CHKERRQ(ierr);
ierr = PetscFree(recv_reqs);CHKERRQ(ierr);
ierr = PetscFree(send_buffer);CHKERRQ(ierr);
ierr = PetscFree(recv_buffer);CHKERRQ(ierr);
ierr = PetscFree(ptrs_buffer);CHKERRQ(ierr);
/* Compute B and B_delta (local actions) */
ierr = PetscMalloc1(pcis->n_neigh,&aux_sums);CHKERRQ(ierr);
ierr = PetscMalloc1(n_local_lambda,&l2g_indices);CHKERRQ(ierr);
ierr = PetscMalloc1(n_local_lambda,&vals_B_delta);CHKERRQ(ierr);
ierr = PetscMalloc1(n_local_lambda,&cols_B_delta);CHKERRQ(ierr);
ierr = PetscMalloc1(n_local_lambda,&scaling_factors);CHKERRQ(ierr);
ierr = ISGetIndices(subset_n,&aux_global_numbering);CHKERRQ(ierr);
n_global_lambda=0;
partial_sum=0;
cum = 0;
for (i=0;i<dual_size;i++) {
n_global_lambda = aux_global_numbering[cum];
j = mat_graph->count[aux_local_numbering_1[i]];
aux_sums[0]=0;
for (s=1;s<j;s++) {
aux_sums[s]=aux_sums[s-1]+j-s+1;
}
array = all_factors[aux_local_numbering_1[i]];
n_neg_values = 0;
while(n_neg_values < j && mat_graph->neighbours_set[aux_local_numbering_1[i]][n_neg_values] < rank) {n_neg_values++;}
n_pos_values = j - n_neg_values;
if (fully_redundant) {
for (s=0;s<n_neg_values;s++) {
l2g_indices [partial_sum+s]=aux_sums[s]+n_neg_values-s-1+n_global_lambda;
cols_B_delta [partial_sum+s]=dual_dofs_boundary_indices[i];
vals_B_delta [partial_sum+s]=-1.0;
scaling_factors[partial_sum+s]=array[s];
}
for (s=0;s<n_pos_values;s++) {
l2g_indices [partial_sum+s+n_neg_values]=aux_sums[n_neg_values]+s+n_global_lambda;
cols_B_delta [partial_sum+s+n_neg_values]=dual_dofs_boundary_indices[i];
vals_B_delta [partial_sum+s+n_neg_values]=1.0;
scaling_factors[partial_sum+s+n_neg_values]=array[s+n_neg_values];
}
partial_sum += j;
} else {
/* l2g_indices and default cols and vals of B_delta */
for (s=0;s<j;s++) {
l2g_indices [partial_sum+s]=n_global_lambda+s;
cols_B_delta [partial_sum+s]=dual_dofs_boundary_indices[i];
vals_B_delta [partial_sum+s]=0.0;
}
/* B_delta */
if ( n_neg_values > 0 ) { /* there's a rank next to me to the left */
vals_B_delta [partial_sum+n_neg_values-1]=-1.0;
}
if ( n_neg_values < j ) { /* there's a rank next to me to the right */
vals_B_delta [partial_sum+n_neg_values]=1.0;
}
/* scaling as in Klawonn-Widlund 1999*/
for (s=0;s<n_neg_values;s++) {
scalar_value = 0.0;
for (k=0;k<s+1;k++) {
scalar_value += array[k];
}
scaling_factors[partial_sum+s] = -scalar_value;
}
for (s=0;s<n_pos_values;s++) {
scalar_value = 0.0;
for (k=s+n_neg_values;k<j;k++) {
scalar_value += array[k];
}
scaling_factors[partial_sum+s+n_neg_values] = scalar_value;
}
partial_sum += j;
}
cum += aux_local_numbering_2[i];
}
ierr = ISRestoreIndices(subset_n,&aux_global_numbering);CHKERRQ(ierr);
ierr = ISDestroy(&subset_mult);CHKERRQ(ierr);
ierr = ISDestroy(&subset_n);CHKERRQ(ierr);
ierr = PetscFree(aux_sums);CHKERRQ(ierr);
ierr = PetscFree(aux_local_numbering_1);CHKERRQ(ierr);
ierr = PetscFree(dual_dofs_boundary_indices);CHKERRQ(ierr);
ierr = PetscFree(all_factors[0]);CHKERRQ(ierr);
ierr = PetscFree(all_factors);CHKERRQ(ierr);
/* Local to global mapping of fetidpmat */
ierr = VecCreate(PETSC_COMM_SELF,&fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSetSizes(fetidpmat_ctx->lambda_local,n_local_lambda,n_local_lambda);CHKERRQ(ierr);
ierr = VecSetType(fetidpmat_ctx->lambda_local,VECSEQ);CHKERRQ(ierr);
ierr = VecCreate(comm,&lambda_global);CHKERRQ(ierr);
ierr = VecSetSizes(lambda_global,PETSC_DECIDE,fetidpmat_ctx->n_lambda);CHKERRQ(ierr);
ierr = VecSetType(lambda_global,VECMPI);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,n_local_lambda,l2g_indices,PETSC_OWN_POINTER,&IS_l2g_lambda);CHKERRQ(ierr);
ierr = VecScatterCreate(fetidpmat_ctx->lambda_local,(IS)0,lambda_global,IS_l2g_lambda,&fetidpmat_ctx->l2g_lambda);CHKERRQ(ierr);
ierr = ISDestroy(&IS_l2g_lambda);CHKERRQ(ierr);
/* Create local part of B_delta */
ierr = MatCreate(PETSC_COMM_SELF,&fetidpmat_ctx->B_delta);CHKERRQ(ierr);
ierr = MatSetSizes(fetidpmat_ctx->B_delta,n_local_lambda,pcis->n_B,n_local_lambda,pcis->n_B);CHKERRQ(ierr);
ierr = MatSetType(fetidpmat_ctx->B_delta,MATSEQAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(fetidpmat_ctx->B_delta,1,NULL);CHKERRQ(ierr);
ierr = MatSetOption(fetidpmat_ctx->B_delta,MAT_IGNORE_ZERO_ENTRIES,PETSC_TRUE);CHKERRQ(ierr);
for (i=0;i<n_local_lambda;i++) {
ierr = MatSetValue(fetidpmat_ctx->B_delta,i,cols_B_delta[i],vals_B_delta[i],INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree(vals_B_delta);CHKERRQ(ierr);
ierr = MatAssemblyBegin(fetidpmat_ctx->B_delta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (fetidpmat_ctx->B_delta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (fully_redundant) {
ierr = MatCreate(PETSC_COMM_SELF,&ScalingMat);CHKERRQ(ierr);
ierr = MatSetSizes(ScalingMat,n_local_lambda,n_local_lambda,n_local_lambda,n_local_lambda);CHKERRQ(ierr);
ierr = MatSetType(ScalingMat,MATSEQAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(ScalingMat,1,NULL);CHKERRQ(ierr);
for (i=0;i<n_local_lambda;i++) {
ierr = MatSetValue(ScalingMat,i,i,scaling_factors[i],INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(ScalingMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (ScalingMat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatMatMult(ScalingMat,fetidpmat_ctx->B_delta,MAT_INITIAL_MATRIX,PETSC_DEFAULT,&fetidpmat_ctx->B_Ddelta);CHKERRQ(ierr);
ierr = MatDestroy(&ScalingMat);CHKERRQ(ierr);
} else {
ierr = MatCreate(PETSC_COMM_SELF,&fetidpmat_ctx->B_Ddelta);CHKERRQ(ierr);
ierr = MatSetSizes(fetidpmat_ctx->B_Ddelta,n_local_lambda,pcis->n_B,n_local_lambda,pcis->n_B);CHKERRQ(ierr);
ierr = MatSetType(fetidpmat_ctx->B_Ddelta,MATSEQAIJ);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(fetidpmat_ctx->B_Ddelta,1,NULL);CHKERRQ(ierr);
for (i=0;i<n_local_lambda;i++) {
ierr = MatSetValue(fetidpmat_ctx->B_Ddelta,i,cols_B_delta[i],scaling_factors[i],INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(fetidpmat_ctx->B_Ddelta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd (fetidpmat_ctx->B_Ddelta,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
ierr = PetscFree(scaling_factors);CHKERRQ(ierr);
ierr = PetscFree(cols_B_delta);CHKERRQ(ierr);
/* Create some vectors needed by fetidp */
ierr = VecDuplicate(pcis->vec1_B,&fetidpmat_ctx->temp_solution_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&fetidpmat_ctx->temp_solution_D);CHKERRQ(ierr);
test_fetidp = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-fetidp_check",&test_fetidp,NULL);CHKERRQ(ierr);
if (test_fetidp && !pcbddc->use_deluxe_scaling) {
PetscReal real_value;
ierr = PetscViewerASCIIGetStdout(comm,&viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"----------FETI_DP TESTS--------------\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"All tests should return zero!\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"FETIDP MAT context in the ");CHKERRQ(ierr);
if (fully_redundant) {
ierr = PetscViewerASCIIPrintf(viewer,"fully redundant case for lagrange multipliers.\n");CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"Non-fully redundant case for lagrange multiplier.\n");CHKERRQ(ierr);
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
/******************************************************************/
/* TEST A/B: Test numbering of global lambda dofs */
/******************************************************************/
ierr = VecDuplicate(fetidpmat_ctx->lambda_local,&test_vec);CHKERRQ(ierr);
ierr = VecSet(lambda_global,1.0);CHKERRQ(ierr);
ierr = VecSet(test_vec,1.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
scalar_value = -1.0;
ierr = VecAXPY(test_vec,scalar_value,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecNorm(test_vec,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = VecDestroy(&test_vec);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"A[%04d]: CHECK glob to loc: % 1.14e\n",rank,real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (fully_redundant) {
ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr);
ierr = VecSet(fetidpmat_ctx->lambda_local,0.5);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecSum(lambda_global,&scalar_value);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"B[%04d]: CHECK loc to glob: % 1.14e\n",rank,PetscRealPart(scalar_value)-fetidpmat_ctx->n_lambda);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
}
/******************************************************************/
/* TEST C: It should holds B_delta*w=0, w\in\widehat{W} */
/* This is the meaning of the B matrix */
/******************************************************************/
ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr);
ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->rctx,pcis->vec1_N,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(matis->rctx,pcis->vec1_global,pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(matis->rctx,pcis->vec1_global,pcis->vec1_N,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_delta */
ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecNorm(lambda_global,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"C[coll]: CHECK infty norm of B_delta*w (w continuous): % 1.14e\n",real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
/******************************************************************/
/* TEST D: It should holds E_Dw = w - P_Dw w\in\widetilde{W} */
/* E_D = R_D^TR */
/* P_D = B_{D,delta}^T B_{delta} */
/* eq.44 Mandel Tezaur and Dohrmann 2005 */
/******************************************************************/
/* compute a random vector in \widetilde{W} */
ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr);
scalar_value = 0.0; /* set zero at vertices */
ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr);
for (i=0;i<n_vertices;i++) { array[vertex_indices[i]]=scalar_value; }
ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr);
/* store w for final comparison */
ierr = VecDuplicate(pcis->vec1_B,&test_vec);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,test_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,test_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Jump operator P_D : results stored in pcis->vec1_B */
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_delta */
ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_Ddelta^T */
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr);
/* Average operator E_D : results stored in pcis->vec2_B */
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = PCBDDCScalingExtension(fetidpmat_ctx->pc,pcis->vec2_B,pcis->vec1_global);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_global,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_global,pcis->vec2_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* test E_D=I-P_D */
scalar_value = 1.0;
ierr = VecAXPY(pcis->vec1_B,scalar_value,pcis->vec2_B);CHKERRQ(ierr);
scalar_value = -1.0;
ierr = VecAXPY(pcis->vec1_B,scalar_value,test_vec);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_B,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = VecDestroy(&test_vec);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"D[%04d] CHECK infty norm of E_D + P_D - I: % 1.14e\n",rank,real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
/******************************************************************/
/* TEST E: It should holds R_D^TP_Dw=0 w\in\widetilde{W} */
/* eq.48 Mandel Tezaur and Dohrmann 2005 */
/******************************************************************/
ierr = VecSetRandom(pcis->vec1_N,NULL);CHKERRQ(ierr);
ierr = VecGetArray(pcis->vec1_N,&array);CHKERRQ(ierr);
scalar_value = 0.0; /* set zero at vertices */
for (i=0;i<n_vertices;i++) { array[vertex_indices[i]]=scalar_value; }
ierr = VecRestoreArray(pcis->vec1_N,&array);CHKERRQ(ierr);
/* Jump operator P_D : results stored in pcis->vec1_B */
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_delta */
ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSet(lambda_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,lambda_global,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* Action of B_Ddelta^T */
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr);
/* scaling */
ierr = PCBDDCScalingExtension(fetidpmat_ctx->pc,pcis->vec1_B,pcis->vec1_global);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_global,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"E[coll]: CHECK infty norm of R^T_D P_D: % 1.14e\n",real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (!fully_redundant) {
/******************************************************************/
/* TEST F: It should holds B_{delta}B^T_{D,delta}=I */
/* Corollary thm 14 Mandel Tezaur and Dohrmann 2005 */
/******************************************************************/
ierr = VecDuplicate(lambda_global,&test_vec);CHKERRQ(ierr);
ierr = VecSetRandom(lambda_global,NULL);CHKERRQ(ierr);
/* Action of B_Ddelta^T */
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,lambda_global,fetidpmat_ctx->lambda_local,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = MatMultTranspose(fetidpmat_ctx->B_Ddelta,fetidpmat_ctx->lambda_local,pcis->vec1_B);CHKERRQ(ierr);
/* Action of B_delta */
ierr = MatMult(fetidpmat_ctx->B_delta,pcis->vec1_B,fetidpmat_ctx->lambda_local);CHKERRQ(ierr);
ierr = VecSet(test_vec,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,test_vec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (fetidpmat_ctx->l2g_lambda,fetidpmat_ctx->lambda_local,test_vec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
scalar_value = -1.0;
ierr = VecAXPY(lambda_global,scalar_value,test_vec);CHKERRQ(ierr);
ierr = VecNorm(lambda_global,NORM_INFINITY,&real_value);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"E[coll]: CHECK infty norm of P^T_D - I: % 1.14e\n",real_value);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = VecDestroy(&test_vec);CHKERRQ(ierr);
}
}
/* final cleanup */
ierr = VecDestroy(&lambda_global);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
TS ts;
SNES snes_alg;
PetscErrorCode ierr;
PetscMPIInt size;
Userctx user;
PetscViewer Xview,Ybusview;
Vec X;
Mat J;
PetscInt i;
ierr = PetscInitialize(&argc,&argv,"petscoptions",help);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size > 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Only for sequential runs");
user.neqs_gen = 9*ngen; /* # eqs. for generator subsystem */
user.neqs_net = 2*nbus; /* # eqs. for network subsystem */
user.neqs_pgrid = user.neqs_gen + user.neqs_net;
/* Create indices for differential and algebraic equations */
PetscInt *idx2;
ierr = PetscMalloc1(7*ngen,&idx2);CHKERRQ(ierr);
for (i=0; i<ngen; i++) {
idx2[7*i] = 9*i; idx2[7*i+1] = 9*i+1; idx2[7*i+2] = 9*i+2; idx2[7*i+3] = 9*i+3;
idx2[7*i+4] = 9*i+6; idx2[7*i+5] = 9*i+7; idx2[7*i+6] = 9*i+8;
}
ierr = ISCreateGeneral(PETSC_COMM_WORLD,7*ngen,idx2,PETSC_COPY_VALUES,&user.is_diff);CHKERRQ(ierr);
ierr = ISComplement(user.is_diff,0,user.neqs_pgrid,&user.is_alg);CHKERRQ(ierr);
ierr = PetscFree(idx2);CHKERRQ(ierr);
/* Read initial voltage vector and Ybus */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"X.bin",FILE_MODE_READ,&Xview);CHKERRQ(ierr);
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"Ybus.bin",FILE_MODE_READ,&Ybusview);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&user.V0);CHKERRQ(ierr);
ierr = VecSetSizes(user.V0,PETSC_DECIDE,user.neqs_net);CHKERRQ(ierr);
ierr = VecLoad(user.V0,Xview);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&user.Ybus);CHKERRQ(ierr);
ierr = MatSetSizes(user.Ybus,PETSC_DECIDE,PETSC_DECIDE,user.neqs_net,user.neqs_net);CHKERRQ(ierr);
ierr = MatSetType(user.Ybus,MATBAIJ);CHKERRQ(ierr);
/* ierr = MatSetBlockSize(user.Ybus,2);CHKERRQ(ierr); */
ierr = MatLoad(user.Ybus,Ybusview);CHKERRQ(ierr);
/* Set run time options */
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"Transient stability fault options","");CHKERRQ(ierr);
{
user.tfaulton = 1.0;
user.tfaultoff = 1.2;
user.Rfault = 0.0001;
user.faultbus = 8;
ierr = PetscOptionsReal("-tfaulton","","",user.tfaulton,&user.tfaulton,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-tfaultoff","","",user.tfaultoff,&user.tfaultoff,NULL);CHKERRQ(ierr);
ierr = PetscOptionsInt("-faultbus","","",user.faultbus,&user.faultbus,NULL);CHKERRQ(ierr);
user.t0 = 0.0;
user.tmax = 5.0;
ierr = PetscOptionsReal("-t0","","",user.t0,&user.t0,NULL);CHKERRQ(ierr);
ierr = PetscOptionsReal("-tmax","","",user.tmax,&user.tmax,NULL);CHKERRQ(ierr);
}
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscViewerDestroy(&Xview);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&Ybusview);CHKERRQ(ierr);
/* Create DMs for generator and network subsystems */
ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,user.neqs_gen,1,1,NULL,&user.dmgen);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(user.dmgen,"dmgen_");CHKERRQ(ierr);
ierr = DMDACreate1d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,user.neqs_net,1,1,NULL,&user.dmnet);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(user.dmnet,"dmnet_");CHKERRQ(ierr);
/* Create a composite DM packer and add the two DMs */
ierr = DMCompositeCreate(PETSC_COMM_WORLD,&user.dmpgrid);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(user.dmpgrid,"pgrid_");CHKERRQ(ierr);
ierr = DMCompositeAddDM(user.dmpgrid,user.dmgen);CHKERRQ(ierr);
ierr = DMCompositeAddDM(user.dmpgrid,user.dmnet);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(user.dmpgrid,&X);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&J);CHKERRQ(ierr);
ierr = MatSetSizes(J,PETSC_DECIDE,PETSC_DECIDE,user.neqs_pgrid,user.neqs_pgrid);CHKERRQ(ierr);
ierr = MatSetFromOptions(J);CHKERRQ(ierr);
ierr = PreallocateJacobian(J,&user);CHKERRQ(ierr);
/* Create matrix to save solutions at each time step */
user.stepnum = 0;
ierr = MatCreateSeqDense(PETSC_COMM_SELF,user.neqs_pgrid+1,1002,NULL,&user.Sol);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create timestepping solver context
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr);
ierr = TSSetProblemType(ts,TS_NONLINEAR);CHKERRQ(ierr);
ierr = TSSetEquationType(ts,TS_EQ_DAE_IMPLICIT_INDEX1);CHKERRQ(ierr);
ierr = TSARKIMEXSetFullyImplicit(ts,PETSC_TRUE);CHKERRQ(ierr);
ierr = TSSetIFunction(ts,NULL,(TSIFunction) IFunction,&user);CHKERRQ(ierr);
ierr = TSSetIJacobian(ts,J,J,(TSIJacobian)IJacobian,&user);CHKERRQ(ierr);
ierr = TSSetApplicationContext(ts,&user);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set initial conditions
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = SetInitialGuess(X,&user);CHKERRQ(ierr);
/* Just to set up the Jacobian structure */
Vec Xdot;
MatStructure flg;
ierr = VecDuplicate(X,&Xdot);CHKERRQ(ierr);
ierr = IJacobian(ts,0.0,X,Xdot,0.0,J,J,&flg,&user);CHKERRQ(ierr);
ierr = VecDestroy(&Xdot);CHKERRQ(ierr);
/* Save initial solution */
PetscScalar *x,*mat;
PetscInt idx=user.stepnum*(user.neqs_pgrid+1);
ierr = MatDenseGetArray(user.Sol,&mat);CHKERRQ(ierr);
ierr = VecGetArray(X,&x);CHKERRQ(ierr);
mat[idx] = 0.0;
ierr = PetscMemcpy(mat+idx+1,x,user.neqs_pgrid*sizeof(PetscScalar));CHKERRQ(ierr);
ierr = MatDenseRestoreArray(user.Sol,&mat);CHKERRQ(ierr);
ierr = VecRestoreArray(X,&x);CHKERRQ(ierr);
user.stepnum++;
ierr = TSSetDuration(ts,1000,user.tfaulton);CHKERRQ(ierr);
ierr = TSSetInitialTimeStep(ts,0.0,0.01);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
ierr = TSSetPostStep(ts,SaveSolution);CHKERRQ(ierr);
user.alg_flg = PETSC_FALSE;
/* Prefault period */
ierr = TSSolve(ts,X);CHKERRQ(ierr);
/* Create the nonlinear solver for solving the algebraic system */
/* Note that although the algebraic system needs to be solved only for
Idq and V, we reuse the entire system including xgen. The xgen
variables are held constant by setting their residuals to 0 and
putting a 1 on the Jacobian diagonal for xgen rows
*/
Vec F_alg;
ierr = VecDuplicate(X,&F_alg);CHKERRQ(ierr);
ierr = SNESCreate(PETSC_COMM_WORLD,&snes_alg);CHKERRQ(ierr);
ierr = SNESSetFunction(snes_alg,F_alg,AlgFunction,&user);CHKERRQ(ierr);
ierr = MatZeroEntries(J);CHKERRQ(ierr);
ierr = SNESSetJacobian(snes_alg,J,J,AlgJacobian,&user);CHKERRQ(ierr);
ierr = SNESSetOptionsPrefix(snes_alg,"alg_");CHKERRQ(ierr);
ierr = SNESSetFromOptions(snes_alg);CHKERRQ(ierr);
/* Apply disturbance - resistive fault at user.faultbus */
/* This is done by adding shunt conductance to the diagonal location
in the Ybus matrix */
PetscInt row_loc,col_loc;
PetscScalar val;
row_loc = 2*user.faultbus; col_loc = 2*user.faultbus+1; /* Location for G */
val = 1/user.Rfault;
ierr = MatSetValues(user.Ybus,1,&row_loc,1,&col_loc,&val,ADD_VALUES);CHKERRQ(ierr);
row_loc = 2*user.faultbus+1; col_loc = 2*user.faultbus; /* Location for G */
val = 1/user.Rfault;
ierr = MatSetValues(user.Ybus,1,&row_loc,1,&col_loc,&val,ADD_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(user.Ybus,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(user.Ybus,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
user.alg_flg = PETSC_TRUE;
/* Solve the algebraic equations */
ierr = SNESSolve(snes_alg,NULL,X);CHKERRQ(ierr);
/* Save fault-on solution */
idx = user.stepnum*(user.neqs_pgrid+1);
ierr = MatDenseGetArray(user.Sol,&mat);CHKERRQ(ierr);
ierr = VecGetArray(X,&x);CHKERRQ(ierr);
mat[idx] = user.tfaulton;
ierr = PetscMemcpy(mat+idx+1,x,user.neqs_pgrid*sizeof(PetscScalar));CHKERRQ(ierr);
ierr = MatDenseRestoreArray(user.Sol,&mat);CHKERRQ(ierr);
ierr = VecRestoreArray(X,&x);CHKERRQ(ierr);
user.stepnum++;
/* Disturbance period */
ierr = TSSetDuration(ts,1000,user.tfaultoff);CHKERRQ(ierr);
ierr = TSSetInitialTimeStep(ts,user.tfaulton,.01);CHKERRQ(ierr);
user.alg_flg = PETSC_FALSE;
ierr = TSSolve(ts,X);CHKERRQ(ierr);
/* Remove the fault */
row_loc = 2*user.faultbus; col_loc = 2*user.faultbus+1;
val = -1/user.Rfault;
ierr = MatSetValues(user.Ybus,1,&row_loc,1,&col_loc,&val,ADD_VALUES);CHKERRQ(ierr);
row_loc = 2*user.faultbus+1; col_loc = 2*user.faultbus;
val = -1/user.Rfault;
ierr = MatSetValues(user.Ybus,1,&row_loc,1,&col_loc,&val,ADD_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(user.Ybus,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(user.Ybus,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatZeroEntries(J);CHKERRQ(ierr);
user.alg_flg = PETSC_TRUE;
/* Solve the algebraic equations */
ierr = SNESSolve(snes_alg,NULL,X);CHKERRQ(ierr);
/* Save tfault off solution */
idx = user.stepnum*(user.neqs_pgrid+1);
ierr = MatDenseGetArray(user.Sol,&mat);CHKERRQ(ierr);
ierr = VecGetArray(X,&x);CHKERRQ(ierr);
mat[idx] = user.tfaultoff;
ierr = PetscMemcpy(mat+idx+1,x,user.neqs_pgrid*sizeof(PetscScalar));CHKERRQ(ierr);
ierr = MatDenseRestoreArray(user.Sol,&mat);CHKERRQ(ierr);
ierr = VecRestoreArray(X,&x);CHKERRQ(ierr);
user.stepnum++;
/* Post-disturbance period */
ierr = TSSetDuration(ts,1000,user.tmax);CHKERRQ(ierr);
ierr = TSSetInitialTimeStep(ts,user.tfaultoff,.01);CHKERRQ(ierr);
user.alg_flg = PETSC_TRUE;
ierr = TSSolve(ts,X);CHKERRQ(ierr);
ierr = MatAssemblyBegin(user.Sol,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(user.Sol,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
Mat A;
PetscScalar *amat;
ierr = MatCreateSeqDense(PETSC_COMM_SELF,user.neqs_pgrid+1,user.stepnum,NULL,&A);CHKERRQ(ierr);
ierr = MatDenseGetArray(user.Sol,&mat);CHKERRQ(ierr);
ierr = MatDenseGetArray(A,&amat);CHKERRQ(ierr);
ierr = PetscMemcpy(amat,mat,(user.stepnum*(user.neqs_pgrid+1))*sizeof(PetscScalar));CHKERRQ(ierr);
ierr = MatDenseRestoreArray(A,&amat);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(user.Sol,&mat);CHKERRQ(ierr);
PetscViewer viewer;
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"out.bin",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
ierr = MatView(A,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = SNESDestroy(&snes_alg);CHKERRQ(ierr);
ierr = VecDestroy(&F_alg);CHKERRQ(ierr);
ierr = MatDestroy(&J);CHKERRQ(ierr);
ierr = MatDestroy(&user.Ybus);CHKERRQ(ierr);
ierr = MatDestroy(&user.Sol);CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
ierr = VecDestroy(&user.V0);CHKERRQ(ierr);
ierr = DMDestroy(&user.dmgen);CHKERRQ(ierr);
ierr = DMDestroy(&user.dmnet);CHKERRQ(ierr);
ierr = DMDestroy(&user.dmpgrid);CHKERRQ(ierr);
ierr = ISDestroy(&user.is_diff);CHKERRQ(ierr);
ierr = ISDestroy(&user.is_alg);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
ierr = PetscFinalize();
return(0);
}
/*
Mm_ratio - ration of grid lines between fine and coarse grids.
*/
int main(int argc,char **argv)
{
PetscErrorCode ierr;
AppCtx user;
PetscInt Npx=PETSC_DECIDE,Npy=PETSC_DECIDE,Npz=PETSC_DECIDE;
PetscMPIInt size,rank;
PetscInt m,n,M,N,i,nrows;
PetscScalar one = 1.0;
PetscReal fill=2.0;
Mat A,A_tmp,P,C,C1,C2;
PetscScalar *array,none = -1.0,alpha;
Vec x,v1,v2,v3,v4;
PetscReal norm,norm_tmp,norm_tmp1,tol=100.*PETSC_MACHINE_EPSILON;
PetscRandom rdm;
PetscBool Test_MatMatMult=PETSC_TRUE,Test_MatPtAP=PETSC_TRUE,Test_3D=PETSC_TRUE,flg;
const PetscInt *ia,*ja;
ierr = PetscInitialize(&argc,&argv,NULL,help);if (ierr) return ierr;
ierr = PetscOptionsGetReal(NULL,NULL,"-tol",&tol,NULL);CHKERRQ(ierr);
user.ratio = 2;
user.coarse.mx = 20; user.coarse.my = 20; user.coarse.mz = 20;
ierr = PetscOptionsGetInt(NULL,NULL,"-Mx",&user.coarse.mx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-My",&user.coarse.my,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Mz",&user.coarse.mz,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-ratio",&user.ratio,NULL);CHKERRQ(ierr);
if (user.coarse.mz) Test_3D = PETSC_TRUE;
user.fine.mx = user.ratio*(user.coarse.mx-1)+1;
user.fine.my = user.ratio*(user.coarse.my-1)+1;
user.fine.mz = user.ratio*(user.coarse.mz-1)+1;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npx",&Npx,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npy",&Npy,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-Npz",&Npz,NULL);CHKERRQ(ierr);
/* Set up distributed array for fine grid */
if (!Test_3D) {
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,Npx,Npy,1,1,NULL,NULL,&user.fine.da);CHKERRQ(ierr);
} else {
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.fine.mx,user.fine.my,user.fine.mz,Npx,Npy,Npz,1,1,NULL,NULL,NULL,&user.fine.da);CHKERRQ(ierr);
}
ierr = DMSetFromOptions(user.fine.da);CHKERRQ(ierr);
ierr = DMSetUp(user.fine.da);CHKERRQ(ierr);
/* Test DMCreateMatrix() */
/*------------------------------------------------------------*/
ierr = DMSetMatType(user.fine.da,MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.fine.da,&A);CHKERRQ(ierr);
ierr = DMSetMatType(user.fine.da,MATBAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(user.fine.da,&C);CHKERRQ(ierr);
ierr = MatConvert(C,MATAIJ,MAT_INITIAL_MATRIX,&A_tmp);CHKERRQ(ierr); /* not work for mpisbaij matrix! */
ierr = MatEqual(A,A_tmp,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"A != C");
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A_tmp);CHKERRQ(ierr);
/*------------------------------------------------------------*/
ierr = MatGetLocalSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetSize(A,&M,&N);CHKERRQ(ierr);
/* if (!rank) printf("A %d, %d\n",M,N); */
/* set val=one to A */
if (size == 1) {
ierr = MatGetRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(A,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(A,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(A,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
} else {
Mat AA,AB;
ierr = MatMPIAIJGetSeqAIJ(A,&AA,&AB,NULL);CHKERRQ(ierr);
ierr = MatGetRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(AA,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(AA,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(AA,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
ierr = MatGetRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
if (flg) {
ierr = MatSeqAIJGetArray(AB,&array);CHKERRQ(ierr);
for (i=0; i<ia[nrows]; i++) array[i] = one;
ierr = MatSeqAIJRestoreArray(AB,&array);CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(AB,0,PETSC_FALSE,PETSC_FALSE,&nrows,&ia,&ja,&flg);CHKERRQ(ierr);
}
/* ierr = MatView(A, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); */
/* Set up distributed array for coarse grid */
if (!Test_3D) {
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,Npx,Npy,1,1,NULL,NULL,&user.coarse.da);CHKERRQ(ierr);
} else {
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,user.coarse.mx,user.coarse.my,user.coarse.mz,Npx,Npy,Npz,
1,1,NULL,NULL,NULL,&user.coarse.da);CHKERRQ(ierr);
}
ierr = DMSetFromOptions(user.coarse.da);CHKERRQ(ierr);
ierr = DMSetUp(user.coarse.da);CHKERRQ(ierr);
/* Create interpolation between the fine and coarse grids */
ierr = DMCreateInterpolation(user.coarse.da,user.fine.da,&P,NULL);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,&m,&n);CHKERRQ(ierr);
ierr = MatGetSize(P,&M,&N);CHKERRQ(ierr);
/* if (!rank) printf("P %d, %d\n",M,N); */
/* Create vectors v1 and v2 that are compatible with A */
ierr = VecCreate(PETSC_COMM_WORLD,&v1);CHKERRQ(ierr);
ierr = MatGetLocalSize(A,&m,NULL);CHKERRQ(ierr);
ierr = VecSetSizes(v1,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(v1);CHKERRQ(ierr);
ierr = VecDuplicate(v1,&v2);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
/* Test MatMatMult(): C = A*P */
/*----------------------------*/
if (Test_MatMatMult) {
ierr = MatDuplicate(A,MAT_COPY_VALUES,&A_tmp);CHKERRQ(ierr);
ierr = MatMatMult(A_tmp,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr);
/* Test MAT_REUSE_MATRIX - reuse symbolic C */
alpha=1.0;
for (i=0; i<2; i++) {
alpha -= 0.1;
ierr = MatScale(A_tmp,alpha);CHKERRQ(ierr);
ierr = MatMatMult(A_tmp,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr);
}
/* Free intermediate data structures created for reuse of C=Pt*A*P */
ierr = MatFreeIntermediateDataStructures(C);CHKERRQ(ierr);
/* Test MatDuplicate() */
/*----------------------------*/
ierr = MatDuplicate(C,MAT_COPY_VALUES,&C1);CHKERRQ(ierr);
ierr = MatDuplicate(C1,MAT_COPY_VALUES,&C2);CHKERRQ(ierr);
ierr = MatDestroy(&C1);CHKERRQ(ierr);
ierr = MatDestroy(&C2);CHKERRQ(ierr);
/* Create vector x that is compatible with P */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,NULL,&n);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
norm = 0.0;
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(P,x,v1);CHKERRQ(ierr);
ierr = MatMult(A_tmp,v1,v2);CHKERRQ(ierr); /* v2 = A*P*x */
ierr = MatMult(C,x,v1);CHKERRQ(ierr); /* v1 = C*x */
ierr = VecAXPY(v1,none,v2);CHKERRQ(ierr);
ierr = VecNorm(v1,NORM_1,&norm_tmp);CHKERRQ(ierr);
ierr = VecNorm(v2,NORM_1,&norm_tmp1);CHKERRQ(ierr);
norm_tmp /= norm_tmp1;
if (norm_tmp > norm) norm = norm_tmp;
}
if (norm >= tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMatMult(), |v1 - v2|/|v2|: %g\n",(double)norm);CHKERRQ(ierr);
}
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A_tmp);CHKERRQ(ierr);
}
/* Test P^T * A * P - MatPtAP() */
/*------------------------------*/
if (Test_MatPtAP) {
ierr = MatPtAP(A,P,MAT_INITIAL_MATRIX,fill,&C);CHKERRQ(ierr);
ierr = MatGetLocalSize(C,&m,&n);CHKERRQ(ierr);
/* Test MAT_REUSE_MATRIX - reuse symbolic C */
alpha=1.0;
for (i=0; i<1; i++) {
alpha -= 0.1;
ierr = MatScale(A,alpha);CHKERRQ(ierr);
ierr = MatPtAP(A,P,MAT_REUSE_MATRIX,fill,&C);CHKERRQ(ierr);
}
/* Free intermediate data structures created for reuse of C=Pt*A*P */
ierr = MatFreeIntermediateDataStructures(C);CHKERRQ(ierr);
/* Test MatDuplicate() */
/*----------------------------*/
ierr = MatDuplicate(C,MAT_COPY_VALUES,&C1);CHKERRQ(ierr);
ierr = MatDuplicate(C1,MAT_COPY_VALUES,&C2);CHKERRQ(ierr);
ierr = MatDestroy(&C1);CHKERRQ(ierr);
ierr = MatDestroy(&C2);CHKERRQ(ierr);
/* Create vector x that is compatible with P */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = MatGetLocalSize(P,&m,&n);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&v3);CHKERRQ(ierr);
ierr = VecSetSizes(v3,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(v3);CHKERRQ(ierr);
ierr = VecDuplicate(v3,&v4);CHKERRQ(ierr);
norm = 0.0;
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(P,x,v1);CHKERRQ(ierr);
ierr = MatMult(A,v1,v2);CHKERRQ(ierr); /* v2 = A*P*x */
ierr = MatMultTranspose(P,v2,v3);CHKERRQ(ierr); /* v3 = Pt*A*P*x */
ierr = MatMult(C,x,v4);CHKERRQ(ierr); /* v3 = C*x */
ierr = VecAXPY(v4,none,v3);CHKERRQ(ierr);
ierr = VecNorm(v4,NORM_1,&norm_tmp);CHKERRQ(ierr);
ierr = VecNorm(v3,NORM_1,&norm_tmp1);CHKERRQ(ierr);
norm_tmp /= norm_tmp1;
if (norm_tmp > norm) norm = norm_tmp;
}
if (norm >= tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatPtAP(), |v3 - v4|/|v3|: %g\n",(double)norm);CHKERRQ(ierr);
}
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = VecDestroy(&v3);CHKERRQ(ierr);
ierr = VecDestroy(&v4);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
}
/* Clean up */
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = VecDestroy(&v1);CHKERRQ(ierr);
ierr = VecDestroy(&v2);CHKERRQ(ierr);
ierr = DMDestroy(&user.fine.da);CHKERRQ(ierr);
ierr = DMDestroy(&user.coarse.da);CHKERRQ(ierr);
ierr = MatDestroy(&P);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscInt main(PetscInt argc,char **args)
{
ptrdiff_t N0=256,N1=256,N2=256,N3=2,dim[4];
fftw_plan bplan,fplan;
fftw_complex *out;
double *in1,*in2;
ptrdiff_t alloc_local,local_n0,local_0_start;
ptrdiff_t local_n1,local_1_start;
PetscInt i,j,indx,n1;
PetscInt size,rank,n,N,*in,N_factor,NM;
PetscScalar *data_fin,value1,one=1.57,zero=0.0;
PetscScalar a,*x_arr,*y_arr,*z_arr,enorm;
Vec fin,fout,fout1,ini,final;
PetscRandom rnd;
PetscErrorCode ierr;
VecScatter vecscat,vecscat1;
IS indx1,indx2;
PetscInt *indx3,k,l,*indx4;
PetscInt low,tempindx,tempindx1;
ierr = PetscInitialize(&argc,&args,(char*)0,help);CHKERRQ(ierr);
#if defined(PETSC_USE_COMPLEX)
SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This example requires real numbers. Your current scalar type is complex");
#endif
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
PetscRandomCreate(PETSC_COMM_WORLD,&rnd);
alloc_local = fftw_mpi_local_size_3d_transposed(N0,N1,N2/2+1,PETSC_COMM_WORLD,&local_n0,&local_0_start,&local_n1,&local_1_start);
/* printf("The value alloc_local is %ld from process %d\n",alloc_local,rank); */
printf("The value local_n0 is %ld from process %d\n",local_n0,rank);
/* printf("The value local_0_start is %ld from process %d\n",local_0_start,rank);*/
/* printf("The value local_n1 is %ld from process %d\n",local_n1,rank); */
/* printf("The value local_1_start is %ld from process %d\n",local_1_start,rank);*/
/* Allocate space for input and output arrays */
in1=(double*)fftw_malloc(sizeof(double)*alloc_local*2);
in2=(double*)fftw_malloc(sizeof(double)*alloc_local*2);
out=(fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
N=2*N0*N1*(N2/2+1);N_factor=N0*N1*N2;
n=2*local_n0*N1*(N2/2+1);n1=local_n1*N0*2*N1;
/* printf("The value N is %d from process %d\n",N,rank); */
/* printf("The value n is %d from process %d\n",n,rank); */
/* printf("The value n1 is %d from process %d\n",n1,rank); */
/* Creating data vector and accompanying array with VeccreateMPIWithArray */
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,n,N,(PetscScalar*)in1,&fin);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,n,N,(PetscScalar*)out,&fout);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,1,n,N,(PetscScalar*)in2,&fout1);CHKERRQ(ierr);
/* VecGetSize(fin,&size); */
/* printf("The size is %d\n",size); */
VecSet(fin,one);
VecSet(fout,zero);
VecSet(fout1,zero);
VecAssemblyBegin(fin);
VecAssemblyEnd(fin);
/* VecView(fin,PETSC_VIEWER_STDOUT_WORLD); */
VecGetArray(fin,&x_arr);
VecGetArray(fout1,&z_arr);
VecGetArray(fout,&y_arr);
fplan=fftw_mpi_plan_dft_r2c_3d(N0,N1,N2,(double*)x_arr,(fftw_complex*)y_arr,PETSC_COMM_WORLD,FFTW_ESTIMATE);
bplan=fftw_mpi_plan_dft_c2r_3d(N0,N1,N2,(fftw_complex*)y_arr,(double*)z_arr,PETSC_COMM_WORLD,FFTW_ESTIMATE);
fftw_execute(fplan);
fftw_execute(bplan);
VecRestoreArray(fin,&x_arr);
VecRestoreArray(fout1,&z_arr);
VecRestoreArray(fout,&y_arr);
/* a = 1.0/(PetscReal)N_factor; */
/* ierr = VecScale(fout1,a);CHKERRQ(ierr); */
VecCreate(PETSC_COMM_WORLD,&ini);
VecCreate(PETSC_COMM_WORLD,&final);
VecSetSizes(ini,local_n0*N1*N2,N_factor);
VecSetSizes(final,local_n0*N1*N2,N_factor);
/* VecSetSizes(ini,PETSC_DECIDE,N_factor); */
/* VecSetSizes(final,PETSC_DECIDE,N_factor); */
VecSetFromOptions(ini);
VecSetFromOptions(final);
if (N2%2==0) NM=N2+2;
else NM=N2+1;
ierr = VecGetOwnershipRange(fin,&low,NULL);
printf("The local index is %d from %d\n",low,rank);
ierr = PetscMalloc(sizeof(PetscInt)*local_n0*N1*N2,&indx3);
ierr = PetscMalloc(sizeof(PetscInt)*local_n0*N1*N2,&indx4);
for (i=0; i<local_n0; i++) {
for (j=0;j<N1;j++) {
for (k=0;k<N2;k++) {
tempindx = i*N1*N2 + j*N2 + k;
tempindx1 = i*N1*NM + j*NM + k;
indx3[tempindx]=local_0_start*N1*N2+tempindx;
indx4[tempindx]=low+tempindx1;
}
/* printf("index3 %d from proc %d is \n",indx3[tempindx],rank); */
/* printf("index4 %d from proc %d is \n",indx4[tempindx],rank); */
}
}
VecGetValues(fin,local_n0*N1*N2,indx4,x_arr);
VecSetValues(ini,local_n0*N1*N2,indx3,x_arr,INSERT_VALUES);
VecAssemblyBegin(ini);
VecAssemblyEnd(ini);
VecGetValues(fout1,local_n0*N1*N2,indx4,y_arr);
VecSetValues(final,local_n0*N1*N2,indx3,y_arr,INSERT_VALUES);
VecAssemblyBegin(final);
VecAssemblyEnd(final);
printf("The local index value is %ld from %d",local_n0*N1*N2,rank);
/*
for (i=0;i<N0;i++) {
for (j=0;j<N1;j++) {
indx=i*N1*NM+j*NM;
ISCreateStride(PETSC_COMM_WORLD,N2,indx,1,&indx1);
indx=i*N1*N2+j*N2;
ISCreateStride(PETSC_COMM_WORLD,N2,indx,1,&indx2);
VecScatterCreate(fin,indx1,ini,indx2,&vecscat);
VecScatterBegin(vecscat,fin,ini,INSERT_VALUES,SCATTER_FORWARD);
VecScatterEnd(vecscat,fin,ini,INSERT_VALUES,SCATTER_FORWARD);
VecScatterCreate(fout1,indx1,final,indx2,&vecscat1);
VecScatterBegin(vecscat1,fout1,final,INSERT_VALUES,SCATTER_FORWARD);
VecScatterEnd(vecscat1,fout1,final,INSERT_VALUES,SCATTER_FORWARD);
}
}
*/
a = 1.0/(PetscReal)N_factor;
ierr = VecScale(fout1,a);CHKERRQ(ierr);
ierr = VecScale(final,a);CHKERRQ(ierr);
VecAssemblyBegin(ini);
VecAssemblyEnd(ini);
VecAssemblyBegin(final);
VecAssemblyEnd(final);
/* VecView(final,PETSC_VIEWER_STDOUT_WORLD); */
ierr = VecAXPY(final,-1.0,ini);CHKERRQ(ierr);
ierr = VecNorm(final,NORM_1,&enorm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Error norm of |x - z| = %e\n",enorm);CHKERRQ(ierr);
fftw_destroy_plan(fplan);
fftw_destroy_plan(bplan);
fftw_free(in1); ierr = VecDestroy(&fin);CHKERRQ(ierr);
fftw_free(out); ierr = VecDestroy(&fout);CHKERRQ(ierr);
fftw_free(in2); ierr = VecDestroy(&fout1);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}