void MultigridSolver_UpdateWorkVectors( MultigridSolver* self ) {
MultigridSolver_Level* level;
//unsigned rowSize, colSize;
PetscInt rowSize, colSize, vecSize;
unsigned l_i;
assert( self && Stg_CheckType( self, MultigridSolver ) );
for( l_i = 0; l_i < self->nLevels - 1; l_i++ ) {
level = self->levels + l_i;
//Matrix_GetLocalSize( MatrixSolver_GetMatrix( level->downSolver ), &rowSize, &colSize );
MatGetLocalSize( level->downSolver->matrix, &rowSize, &colSize );
VecGetLocalSize( level->workSol, &vecSize );
//if( !level->workSol || Vector_GetLocalSize( level->workSol ) != rowSize ) {
if( !level->workSol || vecSize != rowSize ) {
//if( level->workSol )
// Stg_Class_RemoveRef( level->workSol );
//Vector_Duplicate( self->curSolution, (void**)&level->workSol );
//Vector_SetLocalSize( level->workSol, rowSize );
if( level->workSol != PETSC_NULL )
Stg_VecDestroy(&level->workSol );
VecCreate( self->mgData->comm, &level->workSol );
VecSetSizes( level->workSol, rowSize, PETSC_DECIDE );
VecSetFromOptions( level->workSol );
#if( PETSC_VERSION_MAJOR <= 2 && PETSC_VERSION_MINOR >= 3 && PETSC_VERSION_SUBMINOR >= 3 )
VecSetOption( level->workSol, VEC_IGNORE_NEGATIVE_INDICES );
#elif( PETSC_VERSION_MAJOR >= 3 )
VecSetOption( level->workSol, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE );
#endif
}
VecGetLocalSize( level->workRHS, &vecSize );
if( !level->workRHS || /*Vector_GetLocalSize( level->workRHS )*/vecSize != rowSize ) {
//if( level->workRHS )
// Stg_Class_RemoveRef( level->workRHS );
//Vector_Duplicate( self->curSolution, (void**)&level->workRHS );
//Vector_SetLocalSize( level->workRHS, rowSize );
if( level->workRHS != PETSC_NULL )
Stg_VecDestroy(&level->workRHS );
VecCreate( self->mgData->comm, &level->workRHS );
VecSetSizes( level->workRHS, rowSize, PETSC_DECIDE );
VecSetFromOptions( level->workRHS );
#if( PETSC_VERSION_MAJOR <= 2 && PETSC_VERSION_MINOR >= 3 && PETSC_VERSION_SUBMINOR >= 3 )
VecSetOption( level->workRHS, VEC_IGNORE_NEGATIVE_INDICES );
#elif( PETSC_VERSION_MAJOR >= 3 )
VecSetOption( level->workRHS, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE );
#endif
}
}
}
/* these functions were associated with the PETScMatrixSolver class, before it was
* depreciated */
Vec _GetResidual( MGSolver_PETScData* mgData ) {
if( mgData->expiredResidual ) {
VecDuplicate( mgData->curSolution, &mgData->residual );
VecSetFromOptions( mgData->residual );
#if( PETSC_VERSION_MAJOR <= 2 && PETSC_VERSION_MINOR >= 3 && PETSC_VERSION_SUBMINOR >= 3 )
VecSetOption( mgData->residual, VEC_IGNORE_NEGATIVE_INDICES );
#elif( PETSC_VERSION_MAJOR >= 3 )
VecSetOption( mgData->residual, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE );
#endif
MatMult( mgData->matrix, mgData->curSolution, mgData->residual );
VecAYPX( mgData->residual, -1.0, mgData->curRHS );
mgData->expiredResidual = False;
}
return mgData->residual;
}
/*@
MatComputeExplicitOperator - Computes the explicit matrix
Collective on Mat
Input Parameter:
. inmat - the matrix
Output Parameter:
. mat - the explict preconditioned operator
Notes:
This computation is done by applying the operators to columns of the
identity matrix.
Currently, this routine uses a dense matrix format when 1 processor
is used and a sparse format otherwise. This routine is costly in general,
and is recommended for use only with relatively small systems.
Level: advanced
.keywords: Mat, compute, explicit, operator
@*/
PetscErrorCode MatComputeExplicitOperator(Mat inmat,Mat *mat)
{
Vec in,out;
PetscErrorCode ierr;
PetscInt i,m,n,M,N,*rows,start,end;
MPI_Comm comm;
PetscScalar *array,zero = 0.0,one = 1.0;
PetscMPIInt size;
PetscFunctionBegin;
PetscValidHeaderSpecific(inmat,MAT_CLASSID,1);
PetscValidPointer(mat,2);
ierr = PetscObjectGetComm((PetscObject)inmat,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MatGetLocalSize(inmat,&m,&n);CHKERRQ(ierr);
ierr = MatGetSize(inmat,&M,&N);CHKERRQ(ierr);
ierr = MatGetVecs(inmat,&in,&out);CHKERRQ(ierr);
ierr = VecSetOption(in,VEC_IGNORE_OFF_PROC_ENTRIES,PETSC_TRUE);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(out,&start,&end);CHKERRQ(ierr);
ierr = PetscMalloc(m*sizeof(PetscInt),&rows);CHKERRQ(ierr);
for (i=0; i<m; i++) rows[i] = start + i;
ierr = MatCreate(comm,mat);CHKERRQ(ierr);
ierr = MatSetSizes(*mat,m,n,M,N);CHKERRQ(ierr);
if (size == 1) {
ierr = MatSetType(*mat,MATSEQDENSE);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(*mat,NULL);CHKERRQ(ierr);
} else {
ierr = MatSetType(*mat,MATMPIAIJ);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*mat,n,NULL,N-n,NULL);CHKERRQ(ierr);
}
for (i=0; i<N; i++) {
ierr = VecSet(in,zero);CHKERRQ(ierr);
ierr = VecSetValues(in,1,&i,&one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(in);CHKERRQ(ierr);
ierr = VecAssemblyEnd(in);CHKERRQ(ierr);
ierr = MatMult(inmat,in,out);CHKERRQ(ierr);
ierr = VecGetArray(out,&array);CHKERRQ(ierr);
ierr = MatSetValues(*mat,m,rows,1,&i,array,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecRestoreArray(out,&array);CHKERRQ(ierr);
}
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = VecDestroy(&out);CHKERRQ(ierr);
ierr = VecDestroy(&in);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETScVector::config()
{
VecSetFromOptions(_v);
// VecSetUp(_v); // for petsc ver.>3.3
VecGetOwnershipRange(_v, &_start_rank, &_end_rank);
VecGetLocalSize(_v, &_size_loc);
VecGetSize(_v, &_size);
VecSetOption(_v, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE);
}
/*@
STComputeExplicitOperator - Computes the explicit operator associated
to the eigenvalue problem with the specified spectral transformation.
Collective on ST
Input Parameter:
. st - the spectral transform context
Output Parameter:
. mat - the explicit operator
Notes:
This routine builds a matrix containing the explicit operator. For
example, in generalized problems with shift-and-invert spectral
transformation the result would be matrix (A - s B)^-1 B.
This computation is done by applying the operator to columns of the
identity matrix. This is analogous to MatComputeExplicitOperator().
Level: advanced
.seealso: STApply()
@*/
PetscErrorCode STComputeExplicitOperator(ST st,Mat *mat)
{
PetscErrorCode ierr;
Vec in,out;
PetscInt i,M,m,*rows,start,end;
const PetscScalar *array;
PetscScalar one = 1.0;
PetscMPIInt size;
PetscFunctionBegin;
PetscValidHeaderSpecific(st,ST_CLASSID,1);
PetscValidPointer(mat,2);
STCheckMatrices(st,1);
if (st->nmat>2) SETERRQ(PetscObjectComm((PetscObject)st),PETSC_ERR_ARG_WRONGSTATE,"Can only be used with 1 or 2 matrices");
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)st),&size);CHKERRQ(ierr);
ierr = MatGetVecs(st->A[0],&in,&out);CHKERRQ(ierr);
ierr = VecGetSize(out,&M);CHKERRQ(ierr);
ierr = VecGetLocalSize(out,&m);CHKERRQ(ierr);
ierr = VecSetOption(in,VEC_IGNORE_OFF_PROC_ENTRIES,PETSC_TRUE);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(out,&start,&end);CHKERRQ(ierr);
ierr = PetscMalloc1(m,&rows);CHKERRQ(ierr);
for (i=0;i<m;i++) rows[i] = start + i;
ierr = MatCreate(PetscObjectComm((PetscObject)st),mat);CHKERRQ(ierr);
ierr = MatSetSizes(*mat,m,m,M,M);CHKERRQ(ierr);
if (size == 1) {
ierr = MatSetType(*mat,MATSEQDENSE);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(*mat,NULL);CHKERRQ(ierr);
} else {
ierr = MatSetType(*mat,MATMPIAIJ);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*mat,m,NULL,M-m,NULL);CHKERRQ(ierr);
}
for (i=0;i<M;i++) {
ierr = VecSet(in,0.0);CHKERRQ(ierr);
ierr = VecSetValues(in,1,&i,&one,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(in);CHKERRQ(ierr);
ierr = VecAssemblyEnd(in);CHKERRQ(ierr);
ierr = STApply(st,in,out);CHKERRQ(ierr);
ierr = VecGetArrayRead(out,&array);CHKERRQ(ierr);
ierr = MatSetValues(*mat,m,rows,1,&i,array,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(out,&array);CHKERRQ(ierr);
}
ierr = PetscFree(rows);CHKERRQ(ierr);
ierr = VecDestroy(&in);CHKERRQ(ierr);
ierr = VecDestroy(&out);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*mat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt size;
PetscInt n = 9,bs = 3,indices[2],i;
PetscScalar values[6];
Vec x;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_SELF,1,"Must be run with one processor");
/* create vector */
ierr = VecCreate(PETSC_COMM_SELF,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,n);CHKERRQ(ierr);
ierr = VecSetBlockSize(x,bs);CHKERRQ(ierr);
ierr = VecSetType(x,VECSEQ);CHKERRQ(ierr);
for (i=0; i<6; i++) values[i] = 4.0*i;
indices[0] = 0;
indices[1] = 2;
ierr = VecSetValuesBlocked(x,2,indices,values,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
/*
Resulting vector should be 0 4 8 0 0 0 12 16 20
*/
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test insertion with negative indices */
ierr = VecSetOption(x,VEC_IGNORE_NEGATIVE_INDICES,PETSC_TRUE);CHKERRQ(ierr);
for (i=0; i<6; i++) values[i] = -4.0*i;
indices[0] = -1;
indices[1] = 2;
ierr = VecSetValuesBlocked(x,2,indices,values,ADD_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
/*
Resulting vector should be 0 4 8 0 0 0 0 0 0
*/
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
void ReplicatableVector::Replicate(unsigned lo, unsigned hi)
{
// Create a PetSC vector with the array containing the distributed data
Vec distributed_vec;
#if (PETSC_VERSION_MAJOR == 3 && PETSC_VERSION_MINOR >= 3) //PETSc 3.3 or later
//Extra argument is block size
VecCreateMPIWithArray(PETSC_COMM_WORLD, 1, hi-lo, this->GetSize(), &mpData[lo], &distributed_vec);
#else
VecCreateMPIWithArray(PETSC_COMM_WORLD, hi-lo, this->GetSize(), &mpData[lo], &distributed_vec);
#endif
#if (PETSC_VERSION_MAJOR == 3) //PETSc 3.x.x
VecSetOption(distributed_vec, VEC_IGNORE_OFF_PROC_ENTRIES, PETSC_TRUE);
#else
VecSetOption(distributed_vec, VEC_IGNORE_OFF_PROC_ENTRIES);
#endif
// Now do the real replication
ReplicatePetscVector(distributed_vec);
// Clean up
PetscTools::Destroy(distributed_vec);
}
void _SolutionVector_Build( void* solutionVector, void* data ) {
SolutionVector* self = (SolutionVector*)solutionVector;
Journal_DPrintf( self->debug, "In %s - for \"%s\"\n", __func__, self->name );
Stream_IndentBranch( StgFEM_Debug );
/* ensure variables are built */
if( self->feVariable )
Stg_Component_Build( self->feVariable, data, False );
/* Allocate the vector */
VecCreate( self->comm, &self->vector );
VecSetSizes( self->vector, self->feVariable->eqNum->localEqNumsOwnedCount, PETSC_DECIDE );
VecSetFromOptions( self->vector );
#if( PETSC_VERSION_MAJOR <= 2 && PETSC_VERSION_MINOR >= 3 && PETSC_VERSION_SUBMINOR >= 3 )
VecSetOption( self->vector, VEC_IGNORE_NEGATIVE_INDICES );
#elif( PETSC_VERSION_MAJOR >= 3 )
VecSetOption( self->vector, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE );
#endif
Stream_UnIndentBranch( StgFEM_Debug );
}
void _SolutionVector_Build( void* solutionVector, void* data ) {
SolutionVector* self = (SolutionVector*)solutionVector;
Journal_DPrintf( self->debug, "In %s - for \"%s\"\n", __func__, self->name );
Stream_IndentBranch( StgFEM_Debug );
/* ensure variables are built */
if( self->feVariable )
Stg_Component_Build( self->feVariable, data, False );
Journal_Firewall( (self->eqNum!=NULL), NULL, "Solution vector could not be built as provided FeVariable does not appear to have an equation number object.\nPlease contact developers." );
/* Allocate the vector */
VecCreate( self->comm, &self->vector );
VecSetSizes( self->vector, self->eqNum->localEqNumsOwnedCount, PETSC_DECIDE );
VecSetFromOptions( self->vector );
#if( PETSC_VERSION_MAJOR <= 2 && PETSC_VERSION_MINOR >= 3 && PETSC_VERSION_SUBMINOR >= 3 )
VecSetOption( self->vector, VEC_IGNORE_NEGATIVE_INDICES );
#elif( PETSC_VERSION_MAJOR >= 3 )
VecSetOption( self->vector, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE );
#endif
Stream_UnIndentBranch( StgFEM_Debug );
}
void PETScVector::shallowCopy(const PETScVector &v)
{
destroy();
VecDuplicate(v.getRawVector(), &_v);
_start_rank = v._start_rank;
_end_rank = v._end_rank;
_size = v._size;
_size_loc = v._size_loc;
_size_ghosts = v._size_ghosts;
_has_ghost_id = v._has_ghost_id;
VecSetOption(_v, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE);
}
void PETScVector::shallowCopy(const PETScVector &v)
{
destroy();
_v.reset(new PETSc_Vec);
VecDuplicate(*v._v, _v.get());
// TODO can't that be copied from v?
VecGetOwnershipRange(*_v, &_start_rank,&_end_rank);
VecGetLocalSize(*_v, &_size_loc);
VecGetSize(*_v, &_size);
VecSetOption(*_v, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE);
}
/*@C
TaoDefaultComputeGradient - computes the gradient using finite differences.
Collective on Tao
Input Parameters:
+ tao - the Tao context
. X - compute gradient at this point
- dummy - not used
Output Parameters:
. G - Gradient Vector
Options Database Key:
+ -tao_fd_gradient - activates TaoDefaultComputeGradient()
- -tao_fd_delta <delta> - change in X used to calculate finite differences
Level: advanced
Notes:
This routine is slow and expensive, and is not currently optimized
to take advantage of sparsity in the problem. Although
TaoDefaultComputeGradient is not recommended for general use
in large-scale applications, It can be useful in checking the
correctness of a user-provided gradient. Use the tao method TAOTEST
to get an indication of whether your gradient is correct.
This finite difference gradient evaluation can be set using the routine TaoSetGradientRoutine() or by using the command line option -tao_fd_gradient
.seealso: TaoSetGradientRoutine()
@*/
PetscErrorCode TaoDefaultComputeGradient(Tao tao,Vec Xin,Vec G,void *dummy)
{
Vec X;
PetscScalar *g;
PetscReal f, f2;
PetscErrorCode ierr;
PetscInt low,high,N,i;
PetscBool flg;
PetscReal h=.5*PETSC_SQRT_MACHINE_EPSILON;
PetscFunctionBegin;
ierr = PetscOptionsGetReal(((PetscObject)tao)->options,((PetscObject)tao)->prefix,"-tao_fd_delta",&h,&flg);CHKERRQ(ierr);
ierr = VecDuplicate(Xin,&X);CHKERRQ(ierr);
ierr = VecCopy(Xin,X);CHKERRQ(ierr);
ierr = VecGetSize(X,&N);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(X,&low,&high);CHKERRQ(ierr);
ierr = VecSetOption(X,VEC_IGNORE_OFF_PROC_ENTRIES,PETSC_TRUE);CHKERRQ(ierr);
ierr = VecGetArray(G,&g);CHKERRQ(ierr);
for (i=0;i<N;i++) {
ierr = VecSetValue(X,i,-h,ADD_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao,X,&f);CHKERRQ(ierr);
ierr = VecSetValue(X,i,2.0*h,ADD_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
ierr = TaoComputeObjective(tao,X,&f2);CHKERRQ(ierr);
ierr = VecSetValue(X,i,-h,ADD_VALUES);CHKERRQ(ierr);
ierr = VecAssemblyBegin(X);CHKERRQ(ierr);
ierr = VecAssemblyEnd(X);CHKERRQ(ierr);
if (i>=low && i<high) {
g[i-low]=(f2-f)/(2.0*h);
}
}
ierr = VecRestoreArray(G,&g);CHKERRQ(ierr);
ierr = VecDestroy(&X);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETSC_STDCALL vecsetoption_(Vec x,VecOption *op,PetscTruth *flag, int *__ierr ){
*__ierr = VecSetOption(
(Vec)PetscToPointer((x) ),*op,*flag);
}
/* Unlike most finite element applications, IBAMR does assembly on many cells
that are not locally owned; in some cases the processor may own zero finite
element cells but still do assembly on a small number of cells anyway. To
simulate this, this code assembles a PETSc vector by adding contributions
to every entry in the vector on every processor. This causes a deadlock
when we save the communication pattern via
VecSetOption(vec, VEC_SUBSET_OFF_PROC_ENTRIES, PETSC_TRUE).
Contributed-by: David Wells <*****@*****.**>
Petsc developers' notes: this test tests how Petsc knows it can reuse existing communication
pattern. All processes must come to the same conclusion, otherwise deadlock may happen due
to mismatched MPI_Send/Recv. It also tests changing VEC_SUBSET_OFF_PROC_ENTRIES back and forth.
*/
int main(int argc, char **argv)
{
Vec v;
PetscInt i, j, k, *ln, n, rstart;
PetscBool saveCommunicationPattern = PETSC_FALSE;
PetscMPIInt size, rank, p;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, NULL, help);if (ierr) return ierr;
ierr = PetscOptionsGetBool(NULL, NULL, "-save_comm", &saveCommunicationPattern, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = PetscMalloc1(size, &ln);CHKERRQ(ierr);
/* This bug is triggered when one of the local lengths is small. Sometimes in IBAMR this value is actually zero. */
for (p=0; p<size; ++p) ln[p] = 10;
ln[0] = 2;
ierr = PetscPrintf(PETSC_COMM_WORLD, "local lengths are:\n");CHKERRQ(ierr);
ierr = PetscIntView(1, &ln[rank], PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
n = ln[rank];
ierr = VecCreateMPI(MPI_COMM_WORLD, n, PETSC_DECIDE, &v);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(v, &rstart, NULL);CHKERRQ(ierr);
for (k=0; k<5; ++k) { /* 5 iterations of VecAssembly */
PetscReal norm = 0.0;
PetscBool flag = (k == 2) ? PETSC_FALSE : PETSC_TRUE;
PetscInt shift = (k < 2) ? 0 : (k == 2) ? 1 : 0; /* Used to change patterns */
/* If saveCommunicationPattern, let's see what should happen in the 5 iterations:
iter 0: flag is true, and this is the first assebmly, so petsc should keep the
communication pattern built during this assembly.
iter 1: flag is true, reuse the pattern.
iter 2: flag is false, discard/free the pattern built in iter 0; rebuild a new
pattern, but do not keep it after VecAssemblyEnd since the flag is false.
iter 3: flag is true again, this is the new first assembly with a true flag. So
petsc should keep the communication pattern built during this assembly.
iter 4: flag is true, reuse the pattern built in iter 3.
When the vector is destroyed, memory used by the pattern is freed. One can also do it early with a call
VecSetOption(v, VEC_SUBSET_OFF_PROC_ENTRIES, PETSC_FALSE);
*/
if (saveCommunicationPattern) {ierr = VecSetOption(v, VEC_SUBSET_OFF_PROC_ENTRIES, flag);CHKERRQ(ierr);}
ierr = VecSet(v, 0.0);CHKERRQ(ierr);
for (i=0; i<n; ++i) {
PetscScalar val = 1.0;
PetscInt r = rstart + i;
ierr = VecSetValue(v, r, val, ADD_VALUES);CHKERRQ(ierr);
/* do assembly on all other processors too (the 'neighbors') */
{
const PetscMPIInt neighbor = (i+shift) % size; /* Adjust communication patterns between iterations */
const PetscInt nn = ln[neighbor];
PetscInt nrstart = 0;
for (p=0; p<neighbor; ++p) nrstart += ln[p];
for (j=0; j<nn/4; j+= 3) {
PetscScalar val = 0.01;
PetscInt nr = nrstart + j;
ierr = VecSetValue(v, nr, val, ADD_VALUES);CHKERRQ(ierr);
}
}
}
ierr = VecAssemblyBegin(v);CHKERRQ(ierr);
ierr = VecAssemblyEnd(v);CHKERRQ(ierr);
ierr = VecNorm(v, NORM_1, &norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "norm is %g\n", (double)norm);CHKERRQ(ierr);
}
ierr = PetscFree(ln);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
void PETScMGSolver_UpdateWorkVectors( PETScMGSolver* self ) {
PETScMGSolver_Level* level;
PC pc;
//unsigned size;
PetscInt size, vecSize;
PetscErrorCode ec;
unsigned l_i;
assert( self && Stg_CheckType( self, PETScMGSolver ) );
if( self->nLevels == 1 )
return;
ec = KSPGetPC( self->mgData->ksp, &pc );
CheckPETScError( ec );
for( l_i = 0; l_i < self->nLevels; l_i++ ) {
level = self->levels + l_i;
//Matrix_GetLocalSize( level->A, &size, NULL );
MatGetLocalSize( level->A, &size, PETSC_NULL );
if( level->workRes )
VecGetLocalSize( level->workRes, &vecSize );
if( l_i > 0 && (!level->workRes || /*Vector_GetLocalSize( level->workRes )*/vecSize != size) ) {
if( level->workRes )
Stg_VecDestroy(&level->workRes );
// FreeObject( level->workRes );
//Vector_Duplicate( self->curSolution, (void**)&level->workRes );
//Vector_SetLocalSize( level->workRes, size );
//ec = PCMGSetR( pc, l_i, level->workRes->petscVec );
VecCreate( MPI_COMM_WORLD, &level->workRes );
VecSetSizes( level->workRes, size, PETSC_DECIDE );
VecSetFromOptions( level->workRes );
#if( PETSC_VERSION_MAJOR <= 2 && PETSC_VERSION_MINOR >= 3 && PETSC_VERSION_SUBMINOR >= 3 )
VecSetOption( level->workRes, VEC_IGNORE_NEGATIVE_INDICES );
#elif( PETSC_VERSION_MAJOR >= 3 )
VecSetOption( level->workRes, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE );
#endif
ec = PCMGSetR( pc, l_i, level->workRes );
CheckPETScError( ec );
}
if( l_i < self->nLevels - 1 ) {
if( level->workSol )
VecGetLocalSize( level->workSol, &vecSize );
if( !level->workSol || /*Vector_GetLocalSize( level->workSol )*/vecSize != size ) {
if( level->workSol )
Stg_VecDestroy(&level->workSol );
// FreeObject( level->workSol );
//Vector_Duplicate( self->curSolution, (void**)&level->workSol );
//Vector_SetLocalSize( level->workSol, size );
//ec = PCMGSetX( pc, l_i, level->workSol->petscVec );
VecCreate( MPI_COMM_WORLD, &level->workSol );
VecSetSizes( level->workSol, size, PETSC_DECIDE );
VecSetFromOptions( level->workSol );
#if( PETSC_VERSION_MAJOR <= 2 && PETSC_VERSION_MINOR >= 3 && PETSC_VERSION_SUBMINOR >= 3 )
VecSetOption( level->workSol, VEC_IGNORE_NEGATIVE_INDICES );
#elif( PETSC_VERSION_MAJOR >= 3 )
VecSetOption( level->workSol, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE );
#endif
ec = PCMGSetX( pc, l_i, level->workSol );
CheckPETScError( ec );
}
if( level->workRHS )
VecGetLocalSize( level->workRHS, &vecSize );
if( !level->workRHS || /*Vector_GetLocalSize( level->workRHS )*/vecSize != size ) {
if( level->workRHS )
Stg_VecDestroy(&level->workRHS );
// FreeObject( level->workRHS );
//Vector_Duplicate( self->curSolution, (void**)&level->workRHS );
//Vector_SetLocalSize( level->workRHS, size );
//ec = PCMGSetRhs( pc, l_i, level->workRHS->petscVec );
VecCreate( MPI_COMM_WORLD, &level->workRHS );
VecSetSizes( level->workRHS, size, PETSC_DECIDE );
VecSetFromOptions( level->workRHS );
#if( PETSC_VERSION_MAJOR <= 2 && PETSC_VERSION_MINOR >= 3 && PETSC_VERSION_SUBMINOR >= 3 )
VecSetOption( level->workRHS, VEC_IGNORE_NEGATIVE_INDICES );
#elif( PETSC_VERSION_MAJOR >= 3 )
VecSetOption( level->workRHS, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE );
#endif
ec = PCMGSetRhs( pc, l_i, level->workRHS );
CheckPETScError( ec );
}
}
}
}
