PetscErrorCode Monitor(TS ts,PetscInt step,PetscReal time,Vec global,void *ctx)
{
VecScatter scatter;
IS from,to;
PetscInt i,n,*idx,nsteps,maxsteps;
Vec tmp_vec;
PetscErrorCode ierr;
PetscScalar *tmp;
PetscReal maxtime;
Data *data = (Data*)ctx;
PetscReal tfinal = data->tfinal;
PetscFunctionBeginUser;
if (time > tfinal) PetscFunctionReturn(0);
ierr = TSGetTimeStepNumber(ts,&nsteps);CHKERRQ(ierr);
/* display output at selected time steps */
ierr = TSGetDuration(ts, &maxsteps, &maxtime);CHKERRQ(ierr);
if (nsteps % 10 != 0 && time < maxtime) PetscFunctionReturn(0);
/* Get the size of the vector */
ierr = VecGetSize(global,&n);CHKERRQ(ierr);
/* Set the index sets */
ierr = PetscMalloc1(n,&idx);CHKERRQ(ierr);
for (i=0; i<n; i++) idx[i]=i;
/* Create local sequential vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&tmp_vec);CHKERRQ(ierr);
/* Create scatter context */
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,idx,PETSC_COPY_VALUES,&from);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,idx,PETSC_COPY_VALUES,&to);CHKERRQ(ierr);
ierr = VecScatterCreate(global,from,tmp_vec,to,&scatter);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,global,tmp_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,global,tmp_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(tmp_vec,&tmp);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"At t[%D] =%14.2e u= %14.2e at the center \n",nsteps,(double)time,(double)PetscRealPart(tmp[n/2]));CHKERRQ(ierr);
ierr = VecRestoreArray(tmp_vec,&tmp);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter);CHKERRQ(ierr);
ierr = VecDestroy(&tmp_vec);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/* This could restrict auxiliary information to the coarse level.
*/
static PetscErrorCode CoefficientSubDomainRestrictHook(DM dm,VecScatter gscat,VecScatter lscat,DM subdm,void *ctx)
{
Vec c,cc;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = DMGetNamedGlobalVector(dm,"coefficient",&c);CHKERRQ(ierr);
ierr = DMGetNamedGlobalVector(subdm,"coefficient",&cc);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat,c,cc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat,c,cc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = DMRestoreNamedGlobalVector(dm,"coefficient",&c);CHKERRQ(ierr);
ierr = DMRestoreNamedGlobalVector(subdm,"coefficient",&cc);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
DMDANaturalToGlobalBegin - Maps values from a global vector in the "natural" ordering
to a global vector in the PETSc DMDA grid ordering. Must be followed by
DMDANaturalToGlobalEnd() to complete the exchange.
Neighbor-wise Collective on DMDA
Input Parameters:
+ da - the distributed array context
. g - the global vector in a natural ordering
- mode - one of INSERT_VALUES or ADD_VALUES
Output Parameter:
. l - the values in the DMDA ordering
Level: advanced
Notes:
The global and natural vectors used here need not be the same as those
obtained from DMCreateGlobalVector() and DMDACreateNaturalVector(), BUT they
must have the same parallel data layout; they could, for example, be
obtained with VecDuplicate() from the DMDA originating vectors.
.keywords: distributed array, global to local, begin
.seealso: DMDAGlobalToNaturalEnd(), DMDAGlobalToNaturalBegin(), DMLocalToGlobalBegin(), DMDACreate2d(),
DMGlobalToLocalBegin(), DMGlobalToLocalEnd(), DMDACreateNaturalVector()
@*/
PetscErrorCode DMDANaturalToGlobalBegin(DM da,Vec g,InsertMode mode,Vec l)
{
PetscErrorCode ierr;
DM_DA *dd = (DM_DA*)da->data;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidHeaderSpecific(l,VEC_CLASSID,2);
PetscValidHeaderSpecific(g,VEC_CLASSID,4);
if (!dd->gton) {
/* create the scatter context */
ierr = DMDAGlobalToNatural_Create(da);CHKERRQ(ierr);
}
ierr = VecScatterBegin(dd->gton,g,l,mode,SCATTER_REVERSE);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCGAMGGetDataWithGhosts(const Mat Gmat,const PetscInt data_sz,const PetscReal data_in[],PetscInt *a_stride,PetscReal **a_data_out)
{
PetscErrorCode ierr;
PetscMPIInt rank,size;
MPI_Comm comm;
Vec tmp_crds;
Mat_MPIAIJ *mpimat = (Mat_MPIAIJ*)Gmat->data;
PetscInt nnodes,num_ghosts,dir,kk,jj,my0,Iend,nloc;
PetscScalar *data_arr;
PetscReal *datas;
PetscBool isMPIAIJ;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Gmat,&comm);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)Gmat, MATMPIAIJ, &isMPIAIJ);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Gmat, &my0, &Iend);CHKERRQ(ierr);
nloc = Iend - my0;
ierr = VecGetLocalSize(mpimat->lvec, &num_ghosts);CHKERRQ(ierr);
nnodes = num_ghosts + nloc;
*a_stride = nnodes;
ierr = MatGetVecs(Gmat, &tmp_crds, 0);CHKERRQ(ierr);
ierr = PetscMalloc1(data_sz*nnodes, &datas);CHKERRQ(ierr);
for (dir=0; dir<data_sz; dir++) {
/* set local, and global */
for (kk=0; kk<nloc; kk++) {
PetscInt gid = my0 + kk;
PetscScalar crd = (PetscScalar)data_in[dir*nloc + kk]; /* col oriented */
datas[dir*nnodes + kk] = PetscRealPart(crd);
ierr = VecSetValues(tmp_crds, 1, &gid, &crd, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(tmp_crds);CHKERRQ(ierr);
ierr = VecAssemblyEnd(tmp_crds);CHKERRQ(ierr);
/* get ghost datas */
ierr = VecScatterBegin(mpimat->Mvctx,tmp_crds,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(mpimat->Mvctx,tmp_crds,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(mpimat->lvec, &data_arr);CHKERRQ(ierr);
for (kk=nloc,jj=0;jj<num_ghosts;kk++,jj++) datas[dir*nnodes + kk] = PetscRealPart(data_arr[jj]);
ierr = VecRestoreArray(mpimat->lvec, &data_arr);CHKERRQ(ierr);
}
ierr = VecDestroy(&tmp_crds);CHKERRQ(ierr);
*a_data_out = datas;
PetscFunctionReturn(0);
}
//==========================================================================================
PetscErrorCode update_phi(HashTable *El_Table, Vec update, ContData *ctx) {
int elem = 0, myid, i;
HashEntryPtr *buck = El_Table->getbucketptr();
HashEntryPtr currentPtr;
Element *Curr_El;
PetscScalar *update_ptr;
PetscErrorCode ierr;
Vec x_local;
VecScatter *vscat;
PetscInt rank, *num_elem_proc;
vscat = &(ctx->Scatter);
num_elem_proc = ctx->Num_elem_proc;
rank = ctx->rank;
ierr = VecCreateSeq(PETSC_COMM_SELF, num_elem_proc[rank], &x_local);
CHKERRQ(ierr);
ierr = VecScatterBegin(*vscat, update, x_local, INSERT_VALUES, SCATTER_REVERSE);
CHKERRQ(ierr);
ierr = VecScatterEnd(*vscat, update, x_local, INSERT_VALUES, SCATTER_REVERSE);
CHKERRQ(ierr);
ierr = VecGetArray(x_local, &update_ptr);
CHKERRQ(ierr);
for (i = 0; i < El_Table->get_no_of_buckets(); i++)
if (*(buck + i)) {
HashEntryPtr currentPtr = *(buck + i);
while (currentPtr) {
Element* Curr_El = (Element*) (currentPtr->value);
if (Curr_El->get_adapted_flag() > 0) {
*(Curr_El->get_state_vars()) = update_ptr[elem];
elem++;
}
currentPtr = currentPtr->next;
}
}
ierr = VecRestoreArray(x_local, &update_ptr);
CHKERRQ(ierr);
ierr = VecDestroy(&x_local);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatMultAdd_Scatter(Mat A,Vec x,Vec y,Vec z)
{
Mat_Scatter *scatter = (Mat_Scatter*)A->data;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!scatter->scatter) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONGSTATE,"Need to first call MatScatterSetScatter()");
if (z != y) {
ierr = VecCopy(y,z);
CHKERRQ(ierr);
}
ierr = VecScatterBegin(scatter->scatter,x,z,ADD_VALUES,SCATTER_FORWARD);
CHKERRQ(ierr);
ierr = VecScatterEnd(scatter->scatter,x,z,ADD_VALUES,SCATTER_FORWARD);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt n = 5,N;
PetscMPIInt size,rank;
PetscScalar value,zero = 0.0;
Vec x,y;
IS is1,is2;
VecScatter ctx = 0;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* create two vectors */
N = size*n;
ierr = VecCreate(PETSC_COMM_WORLD,&y);CHKERRQ(ierr);
ierr = VecSetSizes(y,PETSC_DECIDE,N);CHKERRQ(ierr);
ierr = VecSetFromOptions(y);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,N,&x);CHKERRQ(ierr);
/* create two index sets */
ierr = ISCreateStride(PETSC_COMM_SELF,n,0,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,n,rank,1,&is2);CHKERRQ(ierr);
value = rank+1;
ierr = VecSet(x,value);CHKERRQ(ierr);
ierr = VecSet(y,zero);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,y,is2,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt rank;
PetscInt n = 5,idx1[2] = {0,3},idx2[2] = {1,4};
PetscScalar one = 1.0,two = 2.0;
Vec x,y;
IS is1,is2;
VecScatter ctx = 0;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* create two vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&y);CHKERRQ(ierr);
/* create two index sets */
ierr = ISCreateGeneral(PETSC_COMM_SELF,2,idx1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,2,idx2,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr);
ierr = VecSet(x,one);CHKERRQ(ierr);
ierr = VecSet(y,two);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,y,is2,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
if (!rank) {VecView(y,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);}
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode DiffIE(Vec originalVec, Vec reducedVec, Vec Distance, Vec nParticlePerCube)
{
// Find the distance between 2 solutions originalVec and reducedVec
PetscInt s,t;
PetscInt n,xstart,xend;
PetscScalar tmp,v,diff;
Vec vec;
VecScatter scat;
IS is;
PetscFunctionBegin;
VecGetOwnershipRange(originalVec,&xstart,&xend);
VecGetSize(reducedVec,&n);
VecCreate(PETSC_COMM_SELF,&vec);
VecSetSizes(vec,PETSC_DECIDE,n);
VecSetFromOptions(vec);
ISCreateStride(PETSC_COMM_WORLD,n,0,1,&is);
VecScatterCreate(reducedVec,PETSC_NULL,vec,is,&scat);
//VecScatterCreateToAll(reducedVec,&scat,&vec);
VecScatterBegin(scat,reducedVec,vec,INSERT_VALUES,SCATTER_FORWARD);
VecScatterEnd(scat,reducedVec,vec,INSERT_VALUES,SCATTER_FORWARD);
for(s=xstart;s<xend;s++)
{
t = ScatIE.FindCube(s);
VecGetValues(vec,1,&t,&tmp);
VecGetValues(originalVec,1,&s,&v);
diff = cabs(v - tmp);
VecSetValues(Distance,1,&t,&diff,ADD_VALUES);
tmp = 1;
VecSetValues(nParticlePerCube,1,&t,&tmp,ADD_VALUES);
}
VecDestroy(&vec);
VecScatterDestroy(&scat);
ISDestroy(&is);
PetscFunctionReturn(0);
}
void PetscVector::copyFromArray( char v[] )
{
int ierr;
Vec sv;
IS is;
VecScatter ctx;
ierr = VecCreateSeq(PETSC_COMM_SELF, n, &sv);
assert(ierr == 0);
{
double * a;
ierr = VecGetArray( sv, &a );
assert( ierr == 0);
int i;
for( i = 0; i < n; i++ ) {
a[i] = v[i];
}
ierr = VecRestoreArray( sv, &a );
assert( ierr == 0);
}
ierr = ISCreateStride(PETSC_COMM_WORLD, n, 0, 1, &is);
assert( ierr == 0);
ierr = VecScatterCreate( sv, is, pv, is, &ctx);
assert( ierr == 0);
ierr = VecScatterBegin( sv, pv,INSERT_VALUES,SCATTER_FORWARD,
ctx);
assert( ierr == 0);
ierr = VecScatterEnd( sv, pv,INSERT_VALUES,SCATTER_FORWARD,
ctx);
assert( ierr == 0);
ierr = VecScatterDestroy(ctx);
assert( ierr == 0);
ierr = ISDestroy( is );
assert(ierr == 0);
ierr = VecDestroy( sv );
assert(ierr == 0);
}
PetscErrorCode TSMonitorLGDMDARay(TS ts, PetscInt step, PetscReal ptime, Vec u, void *ctx)
{
TSMonitorDMDARayCtx *rayctx = (TSMonitorDMDARayCtx *) ctx;
TSMonitorLGCtx lgctx = (TSMonitorLGCtx) rayctx->lgctx;
Vec v = rayctx->ray;
const PetscScalar *a;
PetscInt dim;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = VecScatterBegin(rayctx->scatter, u, v, INSERT_VALUES, SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(rayctx->scatter, u, v, INSERT_VALUES, SCATTER_FORWARD);CHKERRQ(ierr);
if (!step) {
PetscDrawAxis axis;
ierr = PetscDrawLGGetAxis(lgctx->lg, &axis);CHKERRQ(ierr);
ierr = PetscDrawAxisSetLabels(axis, "Solution Ray as function of time", "Time", "Solution");CHKERRQ(ierr);
ierr = VecGetLocalSize(rayctx->ray, &dim);CHKERRQ(ierr);
ierr = PetscDrawLGSetDimension(lgctx->lg, dim);CHKERRQ(ierr);
ierr = PetscDrawLGReset(lgctx->lg);CHKERRQ(ierr);
}
ierr = VecGetArrayRead(v, &a);CHKERRQ(ierr);
#if defined(PETSC_USE_COMPLEX)
{
PetscReal *areal;
PetscInt i,n;
ierr = VecGetLocalSize(v, &n);CHKERRQ(ierr);
ierr = PetscMalloc1(n, &areal);CHKERRQ(ierr);
for (i = 0; i < n; ++i) areal[i] = PetscRealPart(a[i]);
ierr = PetscDrawLGAddCommonPoint(lgctx->lg, ptime, areal);CHKERRQ(ierr);
ierr = PetscFree(areal);CHKERRQ(ierr);
}
#else
ierr = PetscDrawLGAddCommonPoint(lgctx->lg, ptime, a);CHKERRQ(ierr);
#endif
ierr = VecRestoreArrayRead(v, &a);CHKERRQ(ierr);
if (((lgctx->howoften > 0) && (!(step % lgctx->howoften))) || ((lgctx->howoften == -1) && ts->reason)) {
ierr = PetscDrawLGDraw(lgctx->lg);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode FormJacobian(SNES snes,Vec X,Mat *J,Mat *B,MatStructure *flag,void *ptr)
{
AppCtx *user = (AppCtx *) ptr;
PetscErrorCode ierr;
KSP ksp;
PC pc;
PetscBool ismg;
*flag = SAME_NONZERO_PATTERN;
ierr = FormJacobian_Grid(user,&user->fine,X,J,B);CHKERRQ(ierr);
/* create coarse grid jacobian for preconditioner */
ierr = SNESGetKSP(snes,&ksp);CHKERRQ(ierr);
ierr = KSPGetPC(ksp,&pc);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)pc,PCMG,&ismg);CHKERRQ(ierr);
if (ismg) {
ierr = KSPSetOperators(user->ksp_fine,user->fine.J,user->fine.J,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
/* restrict X to coarse grid */
ierr = MatMult(user->R,X,user->coarse.x);CHKERRQ(ierr);
ierr = VecPointwiseMult(user->coarse.x,user->coarse.x,user->Rscale);CHKERRQ(ierr);
/* form Jacobian on coarse grid */
if (user->redundant_build) {
/* get copy of coarse X onto each processor */
ierr = VecScatterBegin(user->tolocalall,user->coarse.x,user->localall,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(user->tolocalall,user->coarse.x,user->localall,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = FormJacobian_Coarse(user,&user->coarse,user->localall,&user->coarse.J,&user->coarse.J);CHKERRQ(ierr);
} else {
/* coarse grid Jacobian computed in parallel */
ierr = FormJacobian_Grid(user,&user->coarse,user->coarse.x,&user->coarse.J,&user->coarse.J);CHKERRQ(ierr);
}
ierr = KSPSetOperators(user->ksp_coarse,user->coarse.J,user->coarse.J,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
return 0;
}
bool SAMpatchPETSc::expandSolution(const SystemVector& solVec,
Vector& dofVec, Real scaleSD) const
{
PETScVector* Bptr = const_cast<PETScVector*>(dynamic_cast<const PETScVector*>(&solVec));
if (!Bptr)
return false;
#ifdef HAVE_MPI
if (adm.isParallel()) {
if (!glob2LocEq) {
std::vector<int> mlgeq(adm.dd.getMLGEQ());
for (auto& it : mlgeq)
--it;
ISCreateGeneral(*adm.getCommunicator(),adm.dd.getMLGEQ().size(),
mlgeq.data(), PETSC_COPY_VALUES, &glob2LocEq);
}
Vec solution;
VecCreateSeq(PETSC_COMM_SELF, Bptr->dim(), &solution);
VecScatter ctx;
VecScatterCreate(Bptr->getVector(), glob2LocEq, solution, nullptr, &ctx);
VecScatterBegin(ctx, Bptr->getVector(), solution, INSERT_VALUES, SCATTER_FORWARD);
VecScatterEnd(ctx, Bptr->getVector(),solution,INSERT_VALUES,SCATTER_FORWARD);
VecScatterDestroy(&ctx);
PetscScalar* data;
VecGetArray(solution, &data);
std::copy(data, data + Bptr->dim(), Bptr->getPtr());
VecDestroy(&solution);
} else
#endif
{
PetscScalar* data;
VecGetArray(Bptr->getVector(), &data);
std::copy(data, data + Bptr->dim(), Bptr->getPtr());
VecRestoreArray(Bptr->getVector(), &data);
}
return this->SAMpatch::expandSolution(solVec, dofVec, scaleSD);
}
PetscErrorCode Monitor(TS ts,PetscInt step,PetscReal time,Vec global,void *ctx)
{
VecScatter scatter;
IS from,to;
PetscInt i,n,*idx;
Vec tmp_vec;
PetscErrorCode ierr;
PetscScalar *tmp;
/* Get the size of the vector */
ierr = VecGetSize(global,&n);CHKERRQ(ierr);
/* Set the index sets */
ierr = PetscMalloc(n*sizeof(PetscInt),&idx);CHKERRQ(ierr);
for(i=0; i<n; i++) idx[i]=i;
/* Create local sequential vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&tmp_vec);CHKERRQ(ierr);
/* Create scatter context */
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,idx,PETSC_COPY_VALUES,&from);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,idx,PETSC_COPY_VALUES,&to);CHKERRQ(ierr);
ierr = VecScatterCreate(global,from,tmp_vec,to,&scatter);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,global,tmp_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,global,tmp_vec,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecGetArray(tmp_vec,&tmp);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"At t =%14.6e u = %14.6e %14.6e %14.6e \n",
time,PetscRealPart(tmp[0]),PetscRealPart(tmp[1]),PetscRealPart(tmp[2]));CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"At t =%14.6e errors = %14.6e %14.6e %14.6e \n",
time,PetscRealPart(tmp[0]-solx(time)),PetscRealPart(tmp[1]-soly(time)),PetscRealPart(tmp[2]-solz(time)));CHKERRQ(ierr);
ierr = VecRestoreArray(tmp_vec,&tmp);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = PetscFree(idx);CHKERRQ(ierr);
ierr = VecDestroy(&tmp_vec);CHKERRQ(ierr);
return 0;
}
//------------------------------------------------------------------------------
// Save solution to file
//------------------------------------------------------------------------------
PetscErrorCode savesol(int nx, double dx, Vec ug)
{
int i, rank;
static int count = 0;
PetscErrorCode ierr;
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
// Gather entire solution on rank=0 process. This is bad thing to do
// in a real application.
VecScatter ctx;
Vec uall;
ierr = VecScatterCreateToZero(ug,&ctx,&uall); CHKERRQ(ierr);
// scatter as many times as you need
ierr = VecScatterBegin(ctx,ug,uall,INSERT_VALUES,SCATTER_FORWARD); CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,ug,uall,INSERT_VALUES,SCATTER_FORWARD); CHKERRQ(ierr);
// destroy scatter context and local vector when no longer needed
ierr = VecScatterDestroy(&ctx); CHKERRQ(ierr);
if(rank==0)
{
PetscScalar *uarray;
ierr = VecGetArray(uall, &uarray); CHKERRQ(ierr);
FILE *f;
f = fopen("sol.dat","w");
for(i=0; i<nx; ++i)
fprintf(f, "%e %e\n", xmin+i*dx, uarray[i]);
fclose(f);
printf("Wrote solution into sol.dat\n");
ierr = VecRestoreArray(uall, &uarray); CHKERRQ(ierr);
if(count==0)
{
// Initial solution is copied to sol0.dat
system("cp sol.dat sol0.dat");
count = 1;
}
}
ierr = VecDestroy(&uall); CHKERRQ(ierr);
return(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt n = 6,idx1[3] = {0,1,2},loc[6] = {0,1,2,3,4,5};
PetscScalar two = 2.0,vals[6] = {10,11,12,13,14,15};
Vec x,y;
IS is1,is2;
VecScatter ctx = 0;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
/* create two vectors */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
/* create two index sets */
ierr = ISCreateStride(PETSC_COMM_SELF,3,0,2,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,3,idx1,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr);
ierr = VecSetValues(x,6,loc,vals,INSERT_VALUES);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"----\n");CHKERRQ(ierr);
ierr = VecSet(y,two);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,y,is2,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DM da,*subda;
PetscInt i,dim=3;
PetscMPIInt size,rank;
Vec v;
Vec slvec,sgvec;
IS *ois,*iis;
VecScatter oscata;
VecScatter *iscat,*oscat,*gscat;
DMDALocalInfo info;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-dim",&dim,NULL);CHKERRQ(ierr);
/* Create distributed array and get vectors */
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (dim == 2) {
ierr = DMDACreate2d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dim == 3) {
ierr = DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-4,-4,-4,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,3,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = DMCreateDomainDecomposition(da,NULL,NULL,&iis,&ois,&subda);CHKERRQ(ierr);
ierr = DMCreateDomainDecompositionScatters(da,1,subda,&iscat,&oscat,&gscat);CHKERRQ(ierr);
{
DMDALocalInfo subinfo;
MatStencil lower,upper;
IS patchis,subpatchis;
Vec smallvec;
Vec largevec;
VecScatter patchscat;
ierr = DMDAGetLocalInfo(subda[0],&subinfo);CHKERRQ(ierr);
lower.i = info.xs;
lower.j = info.ys;
lower.k = info.zs;
upper.i = info.xs+info.xm;
upper.j = info.ys+info.ym;
upper.k = info.zs+info.zm;
/* test the patch IS as a thing to scatter to/from */
ierr = DMDACreatePatchIS(da,&lower,&upper,&patchis);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&smallvec);CHKERRQ(ierr);
ierr = VecSetSizes(smallvec,info.dof*(upper.i - lower.i)*(upper.j - lower.j)*(upper.k - lower.k),PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(smallvec);CHKERRQ(ierr);
ierr = VecScatterCreate(smallvec,NULL,largevec,patchis,&patchscat);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],smallvec);CHKERRQ(ierr);
ierr = VecSet(largevec,0);CHKERRQ(ierr);
ierr = VecScatterBegin(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(patchscat,smallvec,largevec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = ISView(patchis,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecScatterView(patchscat,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = VecView(smallvec,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = VecView(largevec,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&smallvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&largevec);CHKERRQ(ierr);
ierr = ISDestroy(&patchis);CHKERRQ(ierr);
ierr = VecScatterDestroy(&patchscat);CHKERRQ(ierr);
}
/* view the various parts */
{
for (i = 0; i < size; i++) {
if (i == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMView(subda[0],PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMGetLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMGetGlobalVector(da,&v);CHKERRQ(ierr);
/* test filling outer between the big DM and the small ones with the IS scatter*/
ierr = VecScatterCreate(v,ois[0],sgvec,NULL,&oscata);CHKERRQ(ierr);
ierr = FillLocalSubdomain(subda[0],sgvec);CHKERRQ(ierr);
ierr = VecScatterBegin(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscata,sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
/* test the local-to-local scatter */
/* fill up the local subdomain and then add them together */
ierr = FillLocalSubdomain(da,v);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],v,slvec,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test ghost scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(gscat[0],slvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test overlap scattering backwards */
ierr = DMLocalToGlobalBegin(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(subda[0],slvec,ADD_VALUES,sgvec);CHKERRQ(ierr);
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(oscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test interior scattering backwards */
ierr = VecSet(v,0);CHKERRQ(ierr);
ierr = VecScatterBegin(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(iscat[0],sgvec,v,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(v,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test matrix allocation */
for (i = 0; i < size; i++) {
if (i == rank) {
Mat m;
ierr = PetscPrintf(PETSC_COMM_SELF,"Processor %d: \n",i);CHKERRQ(ierr);
ierr = DMSetMatType(subda[0],MATAIJ);CHKERRQ(ierr);
ierr = DMCreateMatrix(subda[0],&m);CHKERRQ(ierr);
ierr = MatView(m,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
ierr = MatDestroy(&m);CHKERRQ(ierr);
}
ierr = MPI_Barrier(PETSC_COMM_WORLD);CHKERRQ(ierr);
}
ierr = DMRestoreLocalVector(subda[0],&slvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(subda[0],&sgvec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(da,&v);CHKERRQ(ierr);
}
ierr = DMDestroy(&subda[0]);CHKERRQ(ierr);
ierr = ISDestroy(&ois[0]);CHKERRQ(ierr);
ierr = ISDestroy(&iis[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&iscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&gscat[0]);CHKERRQ(ierr);
ierr = VecScatterDestroy(&oscata);CHKERRQ(ierr);
ierr = PetscFree(iscat);CHKERRQ(ierr);
ierr = PetscFree(oscat);CHKERRQ(ierr);
ierr = PetscFree(gscat);CHKERRQ(ierr);
ierr = PetscFree(oscata);CHKERRQ(ierr);
ierr = PetscFree(subda);CHKERRQ(ierr);
ierr = PetscFree(ois);CHKERRQ(ierr);
ierr = PetscFree(iis);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode DMCoarsen_DA(DM da, MPI_Comm comm,DM *daref)
{
PetscErrorCode ierr;
PetscInt M,N,P,i;
DM da2;
DM_DA *dd = (DM_DA*)da->data,*dd2;
PetscFunctionBegin;
PetscValidHeaderSpecific(da,DM_CLASSID,1);
PetscValidPointer(daref,3);
if (dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
M = dd->M / dd->coarsen_x;
} else {
M = 1 + (dd->M - 1) / dd->coarsen_x;
}
if (dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
if (dd->dim > 1) {
N = dd->N / dd->coarsen_y;
} else {
N = 1;
}
} else {
N = 1 + (dd->N - 1) / dd->coarsen_y;
}
if (dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0){
if (dd->dim > 2) {
P = dd->P / dd->coarsen_z;
} else {
P = 1;
}
} else {
P = 1 + (dd->P - 1) / dd->coarsen_z;
}
ierr = DMDACreate(((PetscObject)da)->comm,&da2);CHKERRQ(ierr);
ierr = DMSetOptionsPrefix(da2,((PetscObject)da)->prefix);CHKERRQ(ierr);
ierr = DMDASetDim(da2,dd->dim);CHKERRQ(ierr);
ierr = DMDASetSizes(da2,M,N,P);CHKERRQ(ierr);
ierr = DMDASetNumProcs(da2,dd->m,dd->n,dd->p);CHKERRQ(ierr);
ierr = DMDASetBoundaryType(da2,dd->bx,dd->by,dd->bz);CHKERRQ(ierr);
ierr = DMDASetDof(da2,dd->w);CHKERRQ(ierr);
ierr = DMDASetStencilType(da2,dd->stencil_type);CHKERRQ(ierr);
ierr = DMDASetStencilWidth(da2,dd->s);CHKERRQ(ierr);
if (dd->dim == 3) {
PetscInt *lx,*ly,*lz;
ierr = PetscMalloc3(dd->m,PetscInt,&lx,dd->n,PetscInt,&ly,dd->p,PetscInt,&lz);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bz == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_z,dd->p,dd->lz,lz);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,lz);CHKERRQ(ierr);
ierr = PetscFree3(lx,ly,lz);CHKERRQ(ierr);
} else if (dd->dim == 2) {
PetscInt *lx,*ly;
ierr = PetscMalloc2(dd->m,PetscInt,&lx,dd->n,PetscInt,&ly);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->by == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_y,dd->n,dd->ly,ly);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,ly,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree2(lx,ly);CHKERRQ(ierr);
} else if (dd->dim == 1) {
PetscInt *lx;
ierr = PetscMalloc(dd->m*sizeof(PetscInt),&lx);CHKERRQ(ierr);
ierr = DMDACoarsenOwnershipRanges(da,(PetscBool)(dd->bx == DMDA_BOUNDARY_PERIODIC || dd->interptype == DMDA_Q0),dd->s,dd->coarsen_x,dd->m,dd->lx,lx);CHKERRQ(ierr);
ierr = DMDASetOwnershipRanges(da2,lx,PETSC_NULL,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
}
dd2 = (DM_DA*)da2->data;
/* allow overloaded (user replaced) operations to be inherited by refinement clones; why are only some inherited and not all? */
/* da2->ops->createinterpolation = da->ops->createinterpolation; copying this one causes trouble for DMSetVI */
da2->ops->creatematrix = da->ops->creatematrix;
da2->ops->getcoloring = da->ops->getcoloring;
dd2->interptype = dd->interptype;
/* copy fill information if given */
if (dd->dfill) {
ierr = PetscMalloc((dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt),&dd2->dfill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->dfill,dd->dfill,(dd->dfill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
if (dd->ofill) {
ierr = PetscMalloc((dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt),&dd2->ofill);CHKERRQ(ierr);
ierr = PetscMemcpy(dd2->ofill,dd->ofill,(dd->ofill[dd->w]+dd->w+1)*sizeof(PetscInt));CHKERRQ(ierr);
}
/* copy the refine information */
dd2->coarsen_x = dd2->refine_x = dd->coarsen_x;
dd2->coarsen_y = dd2->refine_y = dd->coarsen_y;
dd2->coarsen_z = dd2->refine_z = dd->coarsen_z;
/* copy vector type information */
ierr = PetscFree(da2->vectype);CHKERRQ(ierr);
ierr = PetscStrallocpy(da->vectype,(char**)&da2->vectype);CHKERRQ(ierr);
dd2->lf = dd->lf;
dd2->lj = dd->lj;
da2->leveldown = da->leveldown + 1;
da2->levelup = da->levelup;
ierr = DMSetFromOptions(da2);CHKERRQ(ierr);
ierr = DMSetUp(da2);CHKERRQ(ierr);
ierr = DMViewFromOptions(da2,"-dm_view");CHKERRQ(ierr);
/* inject coordinates if they are set on the fine grid */
if (da->coordinates) {
DM cdaf,cdac;
Vec coordsc,coordsf;
VecScatter inject;
ierr = DMGetCoordinateDM(da,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(da,&coordsf);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(da2,&cdac);CHKERRQ(ierr);
/* force creation of the coordinate vector */
ierr = DMDASetUniformCoordinates(da2,0.0,1.0,0.0,1.0,0.0,1.0);CHKERRQ(ierr);
ierr = DMGetCoordinates(da2,&coordsc);CHKERRQ(ierr);
ierr = DMCreateInjection(cdac,cdaf,&inject);CHKERRQ(ierr);
ierr = VecScatterBegin(inject,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(inject ,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&inject);CHKERRQ(ierr);
}
for (i=0; i<da->bs; i++) {
const char *fieldname;
ierr = DMDAGetFieldName(da,i,&fieldname);CHKERRQ(ierr);
ierr = DMDASetFieldName(da2,i,fieldname);CHKERRQ(ierr);
}
*daref = da2;
PetscFunctionReturn(0);
}
static PetscErrorCode PCBDDCScalingExtension_Deluxe(PC pc, Vec x, Vec y)
{
PC_IS* pcis=(PC_IS*)pc->data;
PC_BDDC* pcbddc=(PC_BDDC*)pc->data;
PCBDDCDeluxeScaling deluxe_ctx = pcbddc->deluxe_ctx;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = VecSet(pcbddc->work_scaling,0.0);CHKERRQ(ierr);
ierr = VecSet(y,0.0);CHKERRQ(ierr);
if (deluxe_ctx->n_simple) { /* scale deluxe vertices using diagonal scaling */
PetscInt i;
const PetscScalar *array_x,*array_D;
PetscScalar *array;
ierr = VecGetArrayRead(x,&array_x);CHKERRQ(ierr);
ierr = VecGetArrayRead(pcis->D,&array_D);CHKERRQ(ierr);
ierr = VecGetArray(pcbddc->work_scaling,&array);CHKERRQ(ierr);
for (i=0;i<deluxe_ctx->n_simple;i++) {
array[deluxe_ctx->idx_simple_B[i]] = array_x[deluxe_ctx->idx_simple_B[i]]*array_D[deluxe_ctx->idx_simple_B[i]];
}
ierr = VecRestoreArray(pcbddc->work_scaling,&array);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(pcis->D,&array_D);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(x,&array_x);CHKERRQ(ierr);
}
/* sequential part : all problems and Schur applications collapsed into a single matrix vector multiplication or a matvec and a solve */
if (deluxe_ctx->seq_mat) {
PetscInt i;
for (i=0;i<deluxe_ctx->seq_n;i++) {
if (deluxe_ctx->change) {
ierr = VecScatterBegin(deluxe_ctx->seq_scctx[i],x,deluxe_ctx->seq_work2[i],INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(deluxe_ctx->seq_scctx[i],x,deluxe_ctx->seq_work2[i],INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (deluxe_ctx->change_with_qr) {
Mat change;
ierr = KSPGetOperators(deluxe_ctx->change[i],&change,NULL);CHKERRQ(ierr);
ierr = MatMultTranspose(change,deluxe_ctx->seq_work2[i],deluxe_ctx->seq_work1[i]);CHKERRQ(ierr);
} else {
ierr = KSPSolve(deluxe_ctx->change[i],deluxe_ctx->seq_work2[i],deluxe_ctx->seq_work1[i]);CHKERRQ(ierr);
}
} else {
ierr = VecScatterBegin(deluxe_ctx->seq_scctx[i],x,deluxe_ctx->seq_work1[i],INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(deluxe_ctx->seq_scctx[i],x,deluxe_ctx->seq_work1[i],INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
ierr = MatMultTranspose(deluxe_ctx->seq_mat[i],deluxe_ctx->seq_work1[i],deluxe_ctx->seq_work2[i]);CHKERRQ(ierr);
if (deluxe_ctx->seq_mat_inv_sum[i]) {
PetscScalar *x;
ierr = VecGetArray(deluxe_ctx->seq_work2[i],&x);CHKERRQ(ierr);
ierr = VecPlaceArray(deluxe_ctx->seq_work1[i],x);CHKERRQ(ierr);
ierr = VecRestoreArray(deluxe_ctx->seq_work2[i],&x);CHKERRQ(ierr);
ierr = MatSolveTranspose(deluxe_ctx->seq_mat_inv_sum[i],deluxe_ctx->seq_work1[i],deluxe_ctx->seq_work2[i]);CHKERRQ(ierr);
ierr = VecResetArray(deluxe_ctx->seq_work1[i]);CHKERRQ(ierr);
}
if (deluxe_ctx->change) {
Mat change;
ierr = KSPGetOperators(deluxe_ctx->change[i],&change,NULL);CHKERRQ(ierr);
ierr = MatMult(change,deluxe_ctx->seq_work2[i],deluxe_ctx->seq_work1[i]);CHKERRQ(ierr);
ierr = VecScatterBegin(deluxe_ctx->seq_scctx[i],deluxe_ctx->seq_work1[i],pcbddc->work_scaling,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(deluxe_ctx->seq_scctx[i],deluxe_ctx->seq_work1[i],pcbddc->work_scaling,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
} else {
ierr = VecScatterBegin(deluxe_ctx->seq_scctx[i],deluxe_ctx->seq_work2[i],pcbddc->work_scaling,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(deluxe_ctx->seq_scctx[i],deluxe_ctx->seq_work2[i],pcbddc->work_scaling,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
}
}
}
/* put local boundary part in global vector */
ierr = VecScatterBegin(pcis->global_to_B,pcbddc->work_scaling,y,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->global_to_B,pcbddc->work_scaling,y,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCScalingSetUp(PC pc)
{
PC_IS* pcis=(PC_IS*)pc->data;
PC_BDDC* pcbddc=(PC_BDDC*)pc->data;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(pc,PC_CLASSID,1);
ierr = PetscLogEventBegin(PC_BDDC_Scaling[pcbddc->current_level],pc,0,0,0);CHKERRQ(ierr);
/* create work vector for the operator */
ierr = VecDestroy(&pcbddc->work_scaling);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcbddc->work_scaling);CHKERRQ(ierr);
/* always rebuild pcis->D */
if (pcis->use_stiffness_scaling) {
PetscScalar *a;
PetscInt i,n;
ierr = MatGetDiagonal(pcbddc->local_mat,pcis->vec1_N);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecAbs(pcis->D);CHKERRQ(ierr);
ierr = VecGetLocalSize(pcis->D,&n);CHKERRQ(ierr);
ierr = VecGetArray(pcis->D,&a);CHKERRQ(ierr);
for (i=0;i<n;i++) if (PetscAbsScalar(a[i])<PETSC_SMALL) a[i] = 1.0;
ierr = VecRestoreArray(pcis->D,&a);CHKERRQ(ierr);
}
ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->D,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->global_to_B,pcis->D,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_global,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->global_to_B,pcis->vec1_global,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPointwiseDivide(pcis->D,pcis->D,pcis->vec1_B);CHKERRQ(ierr);
/* now setup */
if (pcbddc->use_deluxe_scaling) {
if (!pcbddc->deluxe_ctx) {
ierr = PCBDDCScalingCreate_Deluxe(pc);CHKERRQ(ierr);
}
ierr = PCBDDCScalingSetUp_Deluxe(pc);CHKERRQ(ierr);
ierr = PetscObjectComposeFunction((PetscObject)pc,"PCBDDCScalingRestriction_C",PCBDDCScalingRestriction_Deluxe);CHKERRQ(ierr);
ierr = PetscObjectComposeFunction((PetscObject)pc,"PCBDDCScalingExtension_C",PCBDDCScalingExtension_Deluxe);CHKERRQ(ierr);
} else {
ierr = PetscObjectComposeFunction((PetscObject)pc,"PCBDDCScalingRestriction_C",PCBDDCScalingRestriction_Basic);CHKERRQ(ierr);
ierr = PetscObjectComposeFunction((PetscObject)pc,"PCBDDCScalingExtension_C",PCBDDCScalingExtension_Basic);CHKERRQ(ierr);
}
/* test */
if (pcbddc->dbg_flag) {
Mat B0_B = NULL;
Vec B0_Bv = NULL, B0_Bv2 = NULL;
Vec vec2_global;
PetscViewer viewer = pcbddc->dbg_viewer;
PetscReal error;
/* extension -> from local to parallel */
ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr);
ierr = VecSetRandom(pcis->vec1_B,NULL);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->global_to_B,pcis->vec1_B,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_global,&vec2_global);CHKERRQ(ierr);
ierr = VecCopy(pcis->vec1_global,vec2_global);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_global,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->global_to_B,pcis->vec1_global,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (pcbddc->benign_n) {
IS is_dummy;
ierr = ISCreateStride(PETSC_COMM_SELF,pcbddc->benign_n,0,1,&is_dummy);CHKERRQ(ierr);
ierr = MatCreateSubMatrix(pcbddc->benign_B0,is_dummy,pcis->is_B_local,MAT_INITIAL_MATRIX,&B0_B);CHKERRQ(ierr);
ierr = ISDestroy(&is_dummy);CHKERRQ(ierr);
ierr = MatCreateVecs(B0_B,NULL,&B0_Bv);CHKERRQ(ierr);
ierr = VecDuplicate(B0_Bv,&B0_Bv2);CHKERRQ(ierr);
ierr = MatMult(B0_B,pcis->vec1_B,B0_Bv);CHKERRQ(ierr);
}
ierr = PCBDDCScalingExtension(pc,pcis->vec1_B,pcis->vec1_global);CHKERRQ(ierr);
if (pcbddc->benign_saddle_point) {
PetscReal errorl = 0.;
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_global,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->global_to_B,pcis->vec1_global,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (pcbddc->benign_n) {
ierr = MatMult(B0_B,pcis->vec1_B,B0_Bv2);CHKERRQ(ierr);
ierr = VecAXPY(B0_Bv,-1.0,B0_Bv2);CHKERRQ(ierr);
ierr = VecNorm(B0_Bv,NORM_INFINITY,&errorl);CHKERRQ(ierr);
}
ierr = MPI_Allreduce(&errorl,&error,1,MPIU_REAL,MPI_SUM,PetscObjectComm((PetscObject)pc));CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Error benign extension %1.14e\n",error);CHKERRQ(ierr);
}
ierr = VecAXPY(pcis->vec1_global,-1.0,vec2_global);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_global,NORM_INFINITY,&error);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Error scaling extension %1.14e\n",error);CHKERRQ(ierr);
ierr = VecDestroy(&vec2_global);CHKERRQ(ierr);
/* restriction -> from parallel to local */
ierr = VecSet(pcis->vec1_global,0.0);CHKERRQ(ierr);
ierr = VecSetRandom(pcis->vec1_B,NULL);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->global_to_B,pcis->vec1_B,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = PCBDDCScalingRestriction(pc,pcis->vec1_global,pcis->vec1_B);CHKERRQ(ierr);
ierr = VecScale(pcis->vec1_B,-1.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(pcis->global_to_B,pcis->vec1_B,pcis->vec1_global,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecNorm(pcis->vec1_global,NORM_INFINITY,&error);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"Error scaling restriction %1.14e\n",error);CHKERRQ(ierr);
ierr = MatDestroy(&B0_B);CHKERRQ(ierr);
ierr = VecDestroy(&B0_Bv);CHKERRQ(ierr);
ierr = VecDestroy(&B0_Bv2);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(PC_BDDC_Scaling[pcbddc->current_level],pc,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int Smoothing(Vec *x, Vec *y, PetscScalar *cacheScalar, PetscInt *cacheInt , VecScatter *ctx,PetscInt n,DA myDA, PetscInt Istart, PetscInt Iend){
PetscScalar C0,C1,C2;
PetscErrorCode ierr;
PetscInt localsizex;
PetscInt n2,i, j, k;
Vec lvecx,lvecy;
PetscScalar **lvecptx,**lvecpty;
Vec bcvec;
IS isbc;
//VecScatter ctx;
PetscScalar *bcpt1,*bcpt2,*bcpt3,*bcpt4;
localsizex = Iend-Istart;
C0 = 1.0/8;
C1 = 1*1.0/4;
C2 = 1*3.0/16;
n2 = (PetscInt)(n*0.5);
DACreateLocalVector(myDA,&lvecx);
DACreateLocalVector(myDA,&lvecy);
DAGlobalToLocalBegin(myDA,*x,INSERT_VALUES,lvecx);
DAGlobalToLocalEnd(myDA,*x,INSERT_VALUES,lvecx);
VecGetArray2d(lvecx,localsizex+4,n2+4,0,0,&lvecptx);
VecGetArray2d(lvecy,localsizex+4,n2+4,0,0,&lvecpty);
// X direction smoothing
for(j=0;j<n2+4;j++){
for(i=2;i<localsizex+2;i++){
lvecpty[i][j] = C0*lvecptx[i][j]
+C1*lvecptx[i-1][j] +C1*lvecptx[i+1][j]
+C2*lvecptx[i-2][j]+C2*lvecptx[i+2][j];
}
}
VecCreateSeqWithArray(MPI_COMM_SELF,localsizex*4,cacheScalar,&bcvec);
VecScatterBegin(*x,bcvec,INSERT_VALUES,SCATTER_FORWARD,*ctx);
VecScatterEnd(*x,bcvec,INSERT_VALUES,SCATTER_FORWARD,*ctx);
bcpt1 = cacheScalar;
bcpt2 = cacheScalar+localsizex;
bcpt3 = cacheScalar+localsizex*2;
bcpt4 = cacheScalar+localsizex*3;
k= 0;
for(i=2;i<localsizex+2;i++){
lvecpty[i][0]= *(bcpt3+k);
lvecpty[i][1]= *(bcpt4+k);
lvecpty[i][n2+3]= *(bcpt2+k);
lvecpty[i][n2+2]= *(bcpt1+k);
k++;
}
// Y direction smoothing
for(j=2;j<n2+2;j++){
for(i=2;i<localsizex+2;i++){
lvecptx[i][j] = C0*lvecpty[i][j]
+C1*lvecpty[i][j-1] +C1*lvecpty[i][j+1]
+C2*lvecpty[i][j-2] +C2*lvecpty[i][j+2];
}
}
VecRestoreArray2d(lvecy,localsizex+4,n2+4,0,0,&lvecpty);
VecRestoreArray2d(lvecx,localsizex+4,n2+4,0,0,&lvecptx);
DALocalToGlobal(myDA,lvecx,INSERT_VALUES,*x);
VecDestroy(bcvec);
VecDestroy(lvecx);
VecDestroy(lvecy);
return 0;
}
/*
A new Strategy can handle large size problem (more than 4G variables)
*/
int BackwardAverageRL(Vec *x, Vec *y, PetscInt *cacheInt, PetscScalar *cacheScalar, PetscInt n, PetscInt npt, PetscInt pmax, PetscInt Istart, PetscInt Iend,PetscScalar c){
PetscInt rank,size;
PetscErrorCode ierr;
PetscInt i, j, k=0, pi, pj, n2,n4 ,m, puse, pgrid,lx;
PetscInt localsizex,localsizey, rowcount=0;
PetscInt k1,k2,pgrid1,pgrid2;
PetscInt *idy,*idp, *NzindJ;
PetscScalar dx,dy,dx2,dy2,CX,CY;
PetscScalar *pty, *pty0;
IS isx1,isx2,isy1,isy2;
VecScatter ctx1,ctx2;
Vec y0;
Vec x1,x2;
PetscScalar *ptx1,*ptx2;
PetscInt size1,size2,col1,col2;
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
n2 = (PetscInt)(n*0.5);
n4 = (PetscInt)(n*0.25);
dx = 1.0/n;
dy = 1.0/n;
dx2 = dx/2-dx/1e6;
dy2 = dy/2-dy/1e6;
NzindJ = cacheInt; //pmax
idp = cacheInt; //pmax
idy = cacheInt + pmax;
pty0 = cacheScalar ; //pmax
localsizex = Iend-Istart;
localsizey = (PetscInt)(pmax*1.0/(localsizex+1))-2;
if(localsizey>n2){localsizey =n2;}
ierr = VecGetArray(*x,&ptx1);CHKERRQ(ierr);
ptx2 = ptx1;
if(rank< size*0.5){lx = localsizex*n2;}else{lx =0;}
VecCreateMPIWithArray(PETSC_COMM_WORLD,lx,PETSC_DETERMINE,ptx1,&x1);
if(rank< size*0.5){lx = 0;}else{lx = localsizex*n2; }
VecCreateMPIWithArray(PETSC_COMM_WORLD,lx,PETSC_DETERMINE,ptx2,&x2);
VecGetSize(x1,&size1);
VecGetSize(x2,&size2);
col1 = (PetscInt)(size1*1.0/n2);
col2 = (PetscInt)(size2*1.0/n2);
ierr = VecGetArray(*y,&pty);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,(localsizex+1)*(localsizey+1),pty0,&y0);
while(rowcount<n2){
if (n2-rowcount<=localsizey){localsizey =n2-rowcount;}
puse = localsizex*localsizey;
pgrid = (localsizex+1)*(localsizey+1);
k= 0;
k1=0;
k2=0;
for(i=Istart;i<Iend+1;i++){
for(j=rowcount;j<rowcount+localsizey+1;j++){
CX = (PetscScalar)(i*dx);
CY = (PetscScalar)(j*dy);
InverseStandardMap(&CX,&CY,c);
pi = (PetscInt)floor(CX*n);
pj = (PetscInt)floor(CY*n);
if(pj>=n2) {SkewSymmetricPoint(&pi, &pj, n);}
if(pi<col1){
*(NzindJ+k1) = (PetscInt)(n2*pi + pj);
*(idy+k1) = k;
k1++;
}else{
*(NzindJ+pgrid-k2-1) = (PetscInt)(n2*(pi-col1)+pj);
*(idy+pgrid-k2-1) = k;
k2++;
}
k++;
}
}
pgrid1 = k1;
pgrid2 = k2;
ierr = ISCreateGeneralWithArray(PETSC_COMM_SELF,pgrid1,NzindJ,&isx1);CHKERRQ(ierr);
ierr = ISCreateGeneralWithArray(PETSC_COMM_SELF,pgrid2,NzindJ+pgrid1,&isx2);CHKERRQ(ierr);
ierr = ISCreateGeneralWithArray(PETSC_COMM_SELF,pgrid1,idy,&isy1);CHKERRQ(ierr);
ierr = ISCreateGeneralWithArray(PETSC_COMM_SELF,pgrid2,idy+pgrid1,&isy2);CHKERRQ(ierr);
ierr = VecDestroy(y0);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,pgrid,pty0,&y0);CHKERRQ(ierr);
ierr = VecScatterCreate(x1,isx1,y0,isy1,&ctx1);CHKERRQ(ierr);
ierr = VecScatterCreate(x2,isx2,y0,isy2,&ctx2);CHKERRQ(ierr);
ierr = VecScatterBegin(x1,y0,INSERT_VALUES,SCATTER_FORWARD,ctx1);CHKERRQ(ierr);
ierr = VecScatterEnd(x1,y0,INSERT_VALUES,SCATTER_FORWARD,ctx1);CHKERRQ(ierr);
ierr = VecScatterBegin(x2,y0,INSERT_VALUES,SCATTER_FORWARD,ctx2);CHKERRQ(ierr);
ierr = VecScatterEnd(x2,y0,INSERT_VALUES,SCATTER_FORWARD,ctx2);CHKERRQ(ierr);
ierr = VecScatterDestroy(ctx1);
ierr = VecScatterDestroy(ctx2);
ierr = VecGetArray(y0,&pty0);CHKERRQ(ierr);
m = 0;
for(i=0;i<localsizex;i++){
for(j=0;j<localsizey;j++){
*(pty+i*n2+j+rowcount) = (*(pty0+i*(localsizey+1)+j)+
*(pty0+i*(localsizey+1)+j+1)+
*(pty0+(i+1)*(localsizey+1)+j)+
*(pty0+(i+1)*(localsizey+1)+j+1))/4;
m++;
}
}
VecRestoreArray(y0,&pty0);
VecRestoreArray(*y,&pty);
rowcount = rowcount + localsizey;
}
VecDestroy(x1);
VecDestroy(x2);
return 0;
}
uses block index sets\n\n";
#include <petscvec.h>
#undef __FUNCT__
#define __FUNCT__ "main"
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt bs = 1,n = 5,ix0[3] = {5,7,9},ix1[3] = {2,3,4},i,iy0[3] = {1,2,4},iy1[3] = {0,1,3};
PetscMPIInt size,rank;
PetscScalar value;
Vec x,y;
IS isx,isy;
VecScatter ctx = 0,newctx;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (size != 2) SETERRQ(PETSC_COMM_SELF,1,"Must run with 2 processors");
ierr = PetscOptionsGetInt(PETSC_NULL,"-bs",&bs,PETSC_NULL);CHKERRQ(ierr);
n = bs*n;
/* create two vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,size*n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&y);CHKERRQ(ierr);
/* create two index sets */
if (!rank) {
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix0,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy0,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
} else {
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix1,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy1,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
}
/* fill local part of parallel vector */
for (i=n*rank; i<n*(rank+1); i++) {
value = (PetscScalar) i;
ierr = VecSetValues(x,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* fill local part of parallel vector */
for (i=0; i<n; i++) {
value = -(PetscScalar) (i + 100*rank);
ierr = VecSetValues(y,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(y);CHKERRQ(ierr);
ierr = VecAssemblyEnd(y);CHKERRQ(ierr);
ierr = VecScatterCreate(x,isx,y,isy,&ctx);CHKERRQ(ierr);
ierr = VecScatterCopy(ctx,&newctx);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(newctx,y,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(newctx,y,x,INSERT_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterDestroy(&newctx);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISDestroy(&isx);CHKERRQ(ierr);
ierr = ISDestroy(&isy);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode PCISSetUp(PC pc)
{
PC_IS *pcis = (PC_IS*)(pc->data);
Mat_IS *matis;
PetscErrorCode ierr;
PetscBool flg,issbaij;
Vec counter;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)pc->pmat,MATIS,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONG,"Preconditioner type of Neumann Neumman requires matrix of type MATIS");
matis = (Mat_IS*)pc->pmat->data;
pcis->pure_neumann = matis->pure_neumann;
/* get info on mapping */
ierr = PetscObjectReference((PetscObject)matis->mapping);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&pcis->mapping);CHKERRQ(ierr);
pcis->mapping = matis->mapping;
ierr = ISLocalToGlobalMappingGetSize(pcis->mapping,&pcis->n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetInfo(pcis->mapping,&(pcis->n_neigh),&(pcis->neigh),&(pcis->n_shared),&(pcis->shared));CHKERRQ(ierr);
/* Creating local and global index sets for interior and inteface nodes. */
{
PetscInt n_I;
PetscInt *idx_I_local,*idx_B_local,*idx_I_global,*idx_B_global;
PetscInt *array;
PetscInt i,j;
/* Identifying interior and interface nodes, in local numbering */
ierr = PetscMalloc1(pcis->n,&array);CHKERRQ(ierr);
ierr = PetscMemzero(array,pcis->n*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0;i<pcis->n_neigh;i++)
for (j=0;j<pcis->n_shared[i];j++)
array[pcis->shared[i][j]] += 1;
ierr = PetscMalloc1(pcis->n,&idx_I_local);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n,&idx_B_local);CHKERRQ(ierr);
for (i=0, pcis->n_B=0, n_I=0; i<pcis->n; i++) {
if (!array[i]) {
idx_I_local[n_I] = i;
n_I++;
} else {
idx_B_local[pcis->n_B] = i;
pcis->n_B++;
}
}
/* Getting the global numbering */
idx_B_global = idx_I_local + n_I; /* Just avoiding allocating extra memory, since we have vacant space */
idx_I_global = idx_B_local + pcis->n_B;
ierr = ISLocalToGlobalMappingApply(pcis->mapping,pcis->n_B,idx_B_local,idx_B_global);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApply(pcis->mapping,n_I, idx_I_local,idx_I_global);CHKERRQ(ierr);
/* Creating the index sets. */
ierr = ISCreateGeneral(PETSC_COMM_SELF,pcis->n_B,idx_B_local,PETSC_COPY_VALUES, &pcis->is_B_local);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,pcis->n_B,idx_B_global,PETSC_COPY_VALUES,&pcis->is_B_global);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n_I,idx_I_local,PETSC_COPY_VALUES, &pcis->is_I_local);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,n_I,idx_I_global,PETSC_COPY_VALUES,&pcis->is_I_global);CHKERRQ(ierr);
/* Freeing memory and restoring arrays */
ierr = PetscFree(idx_B_local);CHKERRQ(ierr);
ierr = PetscFree(idx_I_local);CHKERRQ(ierr);
ierr = PetscFree(array);CHKERRQ(ierr);
}
/*
Extracting the blocks A_II, A_BI, A_IB and A_BB from A. If the numbering
is such that interior nodes come first than the interface ones, we have
[ | ]
[ A_II | A_IB ]
A = [ | ]
[-----------+------]
[ A_BI | A_BB ]
*/
ierr = MatGetSubMatrix(matis->A,pcis->is_I_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_II);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_B_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_BB);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
if (!issbaij) {
ierr = MatGetSubMatrix(matis->A,pcis->is_I_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_IB);CHKERRQ(ierr);
ierr = MatGetSubMatrix(matis->A,pcis->is_B_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_BI);CHKERRQ(ierr);
} else {
Mat newmat;
ierr = MatConvert(matis->A,MATSEQBAIJ,MAT_INITIAL_MATRIX,&newmat);CHKERRQ(ierr);
ierr = MatGetSubMatrix(newmat,pcis->is_I_local,pcis->is_B_local,MAT_INITIAL_MATRIX,&pcis->A_IB);CHKERRQ(ierr);
ierr = MatGetSubMatrix(newmat,pcis->is_B_local,pcis->is_I_local,MAT_INITIAL_MATRIX,&pcis->A_BI);CHKERRQ(ierr);
ierr = MatDestroy(&newmat);CHKERRQ(ierr);
}
/*
Creating work vectors and arrays
*/
ierr = VecDuplicate(matis->x,&pcis->vec1_N);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_N,&pcis->vec2_N);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pcis->n-pcis->n_B,&pcis->vec1_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec2_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec3_D);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_D,&pcis->vec4_D);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pcis->n_B,&pcis->vec1_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcis->vec2_B);CHKERRQ(ierr);
ierr = VecDuplicate(pcis->vec1_B,&pcis->vec3_B);CHKERRQ(ierr);
ierr = MatCreateVecs(pc->pmat,&pcis->vec1_global,0);CHKERRQ(ierr);
ierr = PetscMalloc1(pcis->n,&pcis->work_N);CHKERRQ(ierr);
/* Creating the scatter contexts */
ierr = VecScatterCreate(pcis->vec1_global,pcis->is_I_global,pcis->vec1_D,(IS)0,&pcis->global_to_D);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_N,pcis->is_B_local,pcis->vec1_B,(IS)0,&pcis->N_to_B);CHKERRQ(ierr);
ierr = VecScatterCreate(pcis->vec1_global,pcis->is_B_global,pcis->vec1_B,(IS)0,&pcis->global_to_B);CHKERRQ(ierr);
/* Creating scaling "matrix" D */
ierr = PetscOptionsGetBool(((PetscObject)pc)->prefix,"-pc_is_use_stiffness_scaling",&pcis->use_stiffness_scaling,NULL);CHKERRQ(ierr);
if (!pcis->D) {
ierr = VecDuplicate(pcis->vec1_B,&pcis->D);CHKERRQ(ierr);
if (!pcis->use_stiffness_scaling) {
ierr = VecSet(pcis->D,pcis->scaling_factor);CHKERRQ(ierr);
} else {
ierr = MatGetDiagonal(matis->A,pcis->vec1_N);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->N_to_B,pcis->vec1_N,pcis->D,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
}
}
ierr = VecCopy(pcis->D,pcis->vec1_B);CHKERRQ(ierr);
ierr = MatCreateVecs(pc->pmat,&counter,0);CHKERRQ(ierr); /* temporary auxiliar vector */
ierr = VecSet(counter,0.0);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,pcis->vec1_B,counter,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,pcis->vec1_B,counter,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterBegin(pcis->global_to_B,counter,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd (pcis->global_to_B,counter,pcis->vec1_B,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecPointwiseDivide(pcis->D,pcis->D,pcis->vec1_B);CHKERRQ(ierr);
ierr = VecDestroy(&counter);CHKERRQ(ierr);
/* See historical note 01, at the bottom of this file. */
/*
Creating the KSP contexts for the local Dirichlet and Neumann problems.
*/
if (pcis->computesolvers) {
PC pc_ctx;
/* Dirichlet */
ierr = KSPCreate(PETSC_COMM_SELF,&pcis->ksp_D);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcis->ksp_D,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(pcis->ksp_D,pcis->A_II,pcis->A_II);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcis->ksp_D,"is_localD_");CHKERRQ(ierr);
ierr = KSPGetPC(pcis->ksp_D,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCLU);CHKERRQ(ierr);
ierr = KSPSetType(pcis->ksp_D,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcis->ksp_D);CHKERRQ(ierr);
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcis->ksp_D);CHKERRQ(ierr);
/* Neumann */
ierr = KSPCreate(PETSC_COMM_SELF,&pcis->ksp_N);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)pcis->ksp_N,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(pcis->ksp_N,matis->A,matis->A);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(pcis->ksp_N,"is_localN_");CHKERRQ(ierr);
ierr = KSPGetPC(pcis->ksp_N,&pc_ctx);CHKERRQ(ierr);
ierr = PCSetType(pc_ctx,PCLU);CHKERRQ(ierr);
ierr = KSPSetType(pcis->ksp_N,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPSetFromOptions(pcis->ksp_N);CHKERRQ(ierr);
{
PetscBool damp_fixed = PETSC_FALSE,
remove_nullspace_fixed = PETSC_FALSE,
set_damping_factor_floating = PETSC_FALSE,
not_damp_floating = PETSC_FALSE,
not_remove_nullspace_floating = PETSC_FALSE;
PetscReal fixed_factor,
floating_factor;
ierr = PetscOptionsGetReal(((PetscObject)pc_ctx)->prefix,"-pc_is_damp_fixed",&fixed_factor,&damp_fixed);CHKERRQ(ierr);
if (!damp_fixed) fixed_factor = 0.0;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_damp_fixed",&damp_fixed,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_remove_nullspace_fixed",&remove_nullspace_fixed,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetReal(((PetscObject)pc_ctx)->prefix,"-pc_is_set_damping_factor_floating",
&floating_factor,&set_damping_factor_floating);CHKERRQ(ierr);
if (!set_damping_factor_floating) floating_factor = 0.0;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_set_damping_factor_floating",&set_damping_factor_floating,NULL);CHKERRQ(ierr);
if (!set_damping_factor_floating) floating_factor = 1.e-12;
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_not_damp_floating",¬_damp_floating,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(((PetscObject)pc_ctx)->prefix,"-pc_is_not_remove_nullspace_floating",¬_remove_nullspace_floating,NULL);CHKERRQ(ierr);
if (pcis->pure_neumann) { /* floating subdomain */
if (!(not_damp_floating)) {
ierr = PCFactorSetShiftType(pc_ctx,MAT_SHIFT_NONZERO);CHKERRQ(ierr);
ierr = PCFactorSetShiftAmount(pc_ctx,floating_factor);CHKERRQ(ierr);
}
if (!(not_remove_nullspace_floating)) {
MatNullSpace nullsp;
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nullsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(pcis->ksp_N,nullsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullsp);CHKERRQ(ierr);
}
} else { /* fixed subdomain */
if (damp_fixed) {
ierr = PCFactorSetShiftType(pc_ctx,MAT_SHIFT_NONZERO);CHKERRQ(ierr);
ierr = PCFactorSetShiftAmount(pc_ctx,floating_factor);CHKERRQ(ierr);
}
if (remove_nullspace_fixed) {
MatNullSpace nullsp;
ierr = MatNullSpaceCreate(PETSC_COMM_SELF,PETSC_TRUE,0,NULL,&nullsp);CHKERRQ(ierr);
ierr = KSPSetNullSpace(pcis->ksp_N,nullsp);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullsp);CHKERRQ(ierr);
}
}
}
/* the vectors in the following line are dummy arguments, just telling the KSP the vector size. Values are not used */
ierr = KSPSetUp(pcis->ksp_N);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode test1_DAInjection3d(PetscInt mx, PetscInt my, PetscInt mz)
{
PetscErrorCode ierr;
DM dac,daf;
PetscViewer vv;
Vec ac,af;
PetscInt periodicity;
DMBoundaryType bx,by,bz;
PetscFunctionBeginUser;
bx = DM_BOUNDARY_NONE;
by = DM_BOUNDARY_NONE;
bz = DM_BOUNDARY_NONE;
periodicity = 0;
ierr = PetscOptionsGetInt(NULL,NULL,"-periodic", &periodicity, NULL);CHKERRQ(ierr);
if (periodicity==1) {
bx = DM_BOUNDARY_PERIODIC;
} else if (periodicity==2) {
by = DM_BOUNDARY_PERIODIC;
} else if (periodicity==3) {
bz = DM_BOUNDARY_PERIODIC;
}
ierr = DMDACreate3d(PETSC_COMM_WORLD, bx,by,bz, DMDA_STENCIL_BOX,mx+1, my+1,mz+1,PETSC_DECIDE, PETSC_DECIDE,PETSC_DECIDE,1, /* 1 dof */
1, /* stencil = 1 */NULL,NULL,NULL,&daf);CHKERRQ(ierr);
ierr = DMSetFromOptions(daf);CHKERRQ(ierr);
ierr = DMSetUp(daf);CHKERRQ(ierr);
ierr = DMCoarsen(daf,MPI_COMM_NULL,&dac);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(dac, -1.0,1.0, -1.0,1.0, -1.0,1.0);CHKERRQ(ierr);
ierr = DMDASetUniformCoordinates(daf, -1.0,1.0, -1.0,1.0, -1.0,1.0);CHKERRQ(ierr);
{
DM cdaf,cdac;
Vec coordsc,coordsf,coordsf2;
Mat inject;
VecScatter vscat;
Mat interp;
PetscReal norm;
ierr = DMGetCoordinateDM(dac,&cdac);CHKERRQ(ierr);
ierr = DMGetCoordinateDM(daf,&cdaf);CHKERRQ(ierr);
ierr = DMGetCoordinates(dac,&coordsc);CHKERRQ(ierr);
ierr = DMGetCoordinates(daf,&coordsf);CHKERRQ(ierr);
ierr = DMCreateInjection(cdac,cdaf,&inject);CHKERRQ(ierr);
ierr = MatScatterGetVecScatter(inject,&vscat);CHKERRQ(ierr);
ierr = VecScatterBegin(vscat,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(vscat ,coordsf,coordsc,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = MatDestroy(&inject);CHKERRQ(ierr);
ierr = DMCreateInterpolation(cdac,cdaf,&interp,NULL);CHKERRQ(ierr);
ierr = VecDuplicate(coordsf,&coordsf2);CHKERRQ(ierr);
ierr = MatInterpolate(interp,coordsc,coordsf2);CHKERRQ(ierr);
ierr = VecAXPY(coordsf2,-1.0,coordsf);CHKERRQ(ierr);
ierr = VecNorm(coordsf2,NORM_MAX,&norm);CHKERRQ(ierr);
/* The fine coordinates are only reproduced in certain cases */
if (!bx && !by && !bz && norm > PETSC_SQRT_MACHINE_EPSILON) {ierr = PetscPrintf(PETSC_COMM_WORLD,"Norm %g\n",(double)norm);CHKERRQ(ierr);}
ierr = VecDestroy(&coordsf2);CHKERRQ(ierr);
ierr = MatDestroy(&interp);CHKERRQ(ierr);
}
if (0) {
ierr = DMCreateGlobalVector(dac,&ac);CHKERRQ(ierr);
ierr = VecZeroEntries(ac);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(daf,&af);CHKERRQ(ierr);
ierr = VecZeroEntries(af);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "dac_7.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(dac, vv);CHKERRQ(ierr);
ierr = VecView(ac, vv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, "daf_7.vtk", &vv);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(vv, PETSC_VIEWER_ASCII_VTK);CHKERRQ(ierr);
ierr = DMView(daf, vv);CHKERRQ(ierr);
ierr = VecView(af, vv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(vv);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&vv);CHKERRQ(ierr);
ierr = VecDestroy(&ac);CHKERRQ(ierr);
ierr = VecDestroy(&af);CHKERRQ(ierr);
}
ierr = DMDestroy(&dac);CHKERRQ(ierr);
ierr = DMDestroy(&daf);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
DMCoarsen_SNESVI - Computes the regular coarsened DM then computes additional information about its inactive set
*/
PetscErrorCode DMCoarsen_SNESVI(DM dm1,MPI_Comm comm,DM *dm2)
{
PetscErrorCode ierr;
PetscContainer isnes;
DM_SNESVI *dmsnesvi1;
Vec finemarked,coarsemarked;
IS inactive;
VecScatter inject;
const PetscInt *index;
PetscInt n,k,cnt = 0,rstart,*coarseindex;
PetscScalar *marked;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)dm1,"VI",(PetscObject *)&isnes);CHKERRQ(ierr);
if (!isnes) SETERRQ(((PetscObject)dm1)->comm,PETSC_ERR_PLIB,"Composed VI data structure is missing");
ierr = PetscContainerGetPointer(isnes,(void**)&dmsnesvi1);CHKERRQ(ierr);
/* get the original coarsen */
ierr = (*dmsnesvi1->coarsen)(dm1,comm,dm2);CHKERRQ(ierr);
/* not sure why this extra reference is needed, but without the dm2 disappears too early */
ierr = PetscObjectReference((PetscObject)*dm2);CHKERRQ(ierr);
/* need to set back global vectors in order to use the original injection */
ierr = DMClearGlobalVectors(dm1);CHKERRQ(ierr);
dm1->ops->createglobalvector = dmsnesvi1->createglobalvector;
ierr = DMCreateGlobalVector(dm1,&finemarked);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(*dm2,&coarsemarked);CHKERRQ(ierr);
/*
fill finemarked with locations of inactive points
*/
ierr = ISGetIndices(dmsnesvi1->inactive,&index);CHKERRQ(ierr);
ierr = ISGetLocalSize(dmsnesvi1->inactive,&n);CHKERRQ(ierr);
ierr = VecSet(finemarked,0.0);CHKERRQ(ierr);
for (k=0;k<n;k++){
ierr = VecSetValue(finemarked,index[k],1.0,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(finemarked);CHKERRQ(ierr);
ierr = VecAssemblyEnd(finemarked);CHKERRQ(ierr);
ierr = DMCreateInjection(*dm2,dm1,&inject);CHKERRQ(ierr);
ierr = VecScatterBegin(inject,finemarked,coarsemarked,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(inject,finemarked,coarsemarked,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&inject);CHKERRQ(ierr);
/*
create index set list of coarse inactive points from coarsemarked
*/
ierr = VecGetLocalSize(coarsemarked,&n);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(coarsemarked,&rstart,PETSC_NULL);CHKERRQ(ierr);
ierr = VecGetArray(coarsemarked,&marked);CHKERRQ(ierr);
for (k=0; k<n; k++) {
if (marked[k] != 0.0) cnt++;
}
ierr = PetscMalloc(cnt*sizeof(PetscInt),&coarseindex);CHKERRQ(ierr);
cnt = 0;
for (k=0; k<n; k++) {
if (marked[k] != 0.0) coarseindex[cnt++] = k + rstart;
}
ierr = VecRestoreArray(coarsemarked,&marked);CHKERRQ(ierr);
ierr = ISCreateGeneral(((PetscObject)coarsemarked)->comm,cnt,coarseindex,PETSC_OWN_POINTER,&inactive);CHKERRQ(ierr);
ierr = DMClearGlobalVectors(dm1);CHKERRQ(ierr);
dm1->ops->createglobalvector = DMCreateGlobalVector_SNESVI;
ierr = DMSetVI(*dm2,inactive);CHKERRQ(ierr);
ierr = VecDestroy(&finemarked);CHKERRQ(ierr);
ierr = VecDestroy(&coarsemarked);CHKERRQ(ierr);
ierr = ISDestroy(&inactive);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
PetscInt rank,size,npt;
PetscErrorCode ierr;
Vec x,y0,tempvec, *vinda,*vindb,*vindc;
PetscInt i,j,k,l,n,p,m,m2,pmax,puse,Istart,Iend,localsize,niter;
PetscScalar dx,dy,dx2,dy2;
PetscScalar *Mixnorm;
PetscInt iter,*iterind,*nind;
FILE *fidoutput;
char fname[50];
PetscViewer socketviewer;
PetscInt withMatlab;
PetscTruth Matlabflag;
PetscLogDouble v1,v2,elapsed_time;
PetscInitialize(&argc,&args,(char *)0,help);
MPI_Comm_size(PETSC_COMM_WORLD,&size);
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPETSC: Petsc Initializes successfully! \n");
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: comm_size is %d \n", size);
ierr = PetscOptionsGetInt(PETSC_NULL,"-withMatlab",&withMatlab,&Matlabflag);CHKERRQ(ierr);
if (Matlabflag == PETSC_FALSE){withMatlab = 0;}else{withMatlab = 1;}
if(withMatlab==1){
// Rank 0 connects to socket, use default socket
PetscViewerSocketOpen(PETSC_COMM_WORLD,0,PETSC_DEFAULT,&socketviewer);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: socket opened! \n");CHKERRQ(ierr);
// Receive n from Matlab
IntReceive(socketviewer, &nind);
n = *nind;
// Receive iter from Matlab
IntReceive(socketviewer, &iterind);
iter = *iterind;
}else{
ierr = PetscOptionsGetInt(PETSC_NULL,"-ngrid",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-niter",&iter,PETSC_NULL);CHKERRQ(ierr);
}
/////////////////////////////////////////////////////////////////////////////////////
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of grid is %d \n", n);
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: number of iteration is %d \n", iter);
Mixnorm = malloc(iter*sizeof(PetscScalar));
dx = 1.0/n;
dy = 1.0/n;
dx2 = dx/2-dx/1e6;
dy2 = dy/2-dy/1e6;
npt = 5;
pmax = 4e6;
puse = pmax;
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated buffer size (per processer) %f Mbytes \n", pmax*1.0/1e6*8*16 );
ierr = PetscPrintf(PETSC_COMM_WORLD,"PETSC: estimated variable size %f Mbytes\n", 1.0*n*n/1e6*8*2);
/////////////////////////////////////////////////////////////////////////////////////
ierr = VecCreateMPI(PETSC_COMM_WORLD,PETSC_DECIDE ,n,&tempvec);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(tempvec,&Istart,&Iend);CHKERRQ(ierr);
localsize = Iend-Istart;
ierr = VecDestroy(tempvec);CHKERRQ(ierr);
/////////////////////////////////////////////////////////////////////////////////////
// Create initial vector
Vec x0;
PetscScalar *x0array;
x0array = malloc((localsize)*n*sizeof(PetscScalar));
k = 0;
for(i=Istart;i<Iend;i++){
for(j=0;j<n;j++){
*(x0array+k) = cos(2*M_PI*(dx/2+i*dx));
//*(x0array+k) = cos(2*M_PI*(dy/2+j*dy));
k++;
}
}
ierr = VecCreateMPIWithArray(PETSC_COMM_WORLD,n*localsize,PETSC_DECIDE,x0array,&x0);CHKERRQ(ierr);
ierr = VecDuplicate(x0,&x);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,pmax*npt,&y0);CHKERRQ(ierr);
ierr = VecNorm(x0,NORM_2,Mixnorm); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"PETSC: initial norm= %f \n",*(Mixnorm+0)/n );
///////////////////////////////////////////////////////////////////////////
// Map Center Points
PetscInt *NzindJ,*idx,*idy,*idp;
PetscScalar *CenterX,*CenterY,*VecVal,*pty;
PetscScalar *ShiftX,*ShiftY,CX,CY, *yarray;
IS isx,isy;
VecScatter ctx;
CenterX = malloc(npt*sizeof(PetscScalar));
CenterY = malloc(npt*sizeof(PetscScalar));
ShiftX = malloc(npt*sizeof(PetscScalar));
ShiftY = malloc(npt*sizeof(PetscScalar));
VecVal = malloc(npt*sizeof(PetscScalar));
yarray = malloc(pmax*sizeof(PetscScalar));
NzindJ = malloc(pmax*npt*sizeof(PetscInt));
idx = malloc(pmax*npt*sizeof(PetscInt));
idy = malloc(pmax*npt*sizeof(PetscInt));
idp = malloc(pmax*sizeof(PetscInt));
*(ShiftX+0) = 0;
*(ShiftY+0) = 0;
*(ShiftX+1) = -dx2;
*(ShiftY+1) = -dy2;
*(ShiftX+2) = dx2;
*(ShiftY+2) = -dy2;
*(ShiftX+3) = -dx2;
*(ShiftY+3) = dy2;
*(ShiftX+4) = dy2;
*(ShiftY+4) = dx2;
//*(ShiftX+5) = 0;
//*(ShiftY+5) = -dy2;
//*(ShiftX+6) = -dx2;
//*(ShiftY+6) = 0;
//*(ShiftX+7) = dx2;
//*(ShiftY+7) = 0;
//*(ShiftX+8) = 0;
//*(ShiftY+9) = dy2;
for(i=0;i<npt*pmax;i++){ *(idy+i)=i; }
ISCreateGeneralWithArray(PETSC_COMM_SELF,npt*pmax,idy,&isy);
vinda = &x0;
vindb = &x;
sprintf(fname, "mixnorm_%d_%d",n,iter);
ierr =PetscPrintf(PETSC_COMM_WORLD,"\n iter norm time unit time\n");CHKERRQ(ierr);
ierr =PetscFOpen(PETSC_COMM_WORLD,fname,"w",&fidoutput);CHKERRQ(ierr);
for(niter=0;niter<iter;niter++){
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
l = 0; p = 0;
if (n*localsize-l<=pmax){puse = n*localsize-l;}else{puse=pmax;}
for(i=Istart;i<Iend;i++){
for(j=0;j<n;j++){
CX = dx2+i*dx;
CY = dy2+j*dy;
for(k=0;k<npt;k++){
*(CenterX+k) = CX + *(ShiftX+k);
*(CenterY+k) = CY + *(ShiftY+k);
InverseStandardMap((CenterX+k),(CenterY+k));
*(NzindJ+p*npt +k) = floor(*(CenterX+k)*n)*n + floor(*(CenterY+k)*n);
}
*(idp+p) = Istart*n+ l;
if(p>=puse-1){
ierr = ISCreateGeneralWithArray(PETSC_COMM_WORLD,npt*puse,NzindJ,&isx);CHKERRQ(ierr);
for(m=0;m<npt*puse;m++){ *(idy+m)=m; }
ierr = ISCreateGeneralWithArray(PETSC_COMM_SELF,npt*puse,idy,&isy);CHKERRQ(ierr);
ierr = VecScatterCreate(*vinda,isx,y0,isy,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(*vinda,y0,INSERT_VALUES,SCATTER_FORWARD,ctx);CHKERRQ(ierr);
ierr = VecScatterEnd(*vinda,y0,INSERT_VALUES,SCATTER_FORWARD,ctx);CHKERRQ(ierr);
ierr = VecScatterDestroy(ctx);
ierr = VecGetArray(y0,&pty);CHKERRQ(ierr);
for(m=0;m<puse;m++){
for(m2=0;m2<npt;m2++){
*(yarray+m) = *(yarray+m)+*(pty+m*npt+m2);
}
*(yarray+m) = *(yarray+m)/npt;
}
VecRestoreArray(y0,&pty);
VecSetValues(*vindb,puse,idp,yarray,INSERT_VALUES);
for(m=0;m<pmax;m++){*(yarray+m) = 0; }
p = 0;
if (n*localsize-l<=pmax){puse = n*localsize-l-1;}else{puse=pmax;}
}else{p++;}
l++;
}
}
VecAssemblyBegin(*vindb);
VecAssemblyEnd(*vindb);
vindc = vindb;
vindb = vinda;
vinda = vindc;
//ierr = VecCopy(x,x0);CHKERRQ(ierr);
ierr = VecNorm(*vinda,NORM_2,Mixnorm+niter); CHKERRQ(ierr);
*(Mixnorm+niter) = *(Mixnorm+niter)/n;
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
elapsed_time = v2 - v1;
PetscPrintf(PETSC_COMM_WORLD," %d %f %f %f \n",niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
PetscFPrintf(PETSC_COMM_WORLD,fidoutput," %d %f %f %f\n"
,niter,*(Mixnorm+niter),elapsed_time,elapsed_time/n/n*1e6 );
}
PetscFClose(PETSC_COMM_WORLD,fidoutput);
///////////////////////////////////////////////////////////////////////////
if(withMatlab==1){
VecView(x0,socketviewer);
PetscScalarView(iter,Mixnorm,socketviewer);
}
free(CenterX);
free(CenterY);
free(ShiftX);
free(ShiftY);
free(x0array);
free(idx);
free(idy);
free(idp);
free(yarray);
free(NzindJ);
free(Mixnorm);
ierr = VecDestroy(x0);CHKERRQ(ierr);
ierr = VecDestroy(x);CHKERRQ(ierr);
ierr = VecDestroy(y0);CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Done!");
//////////////////////////////////////////////////////////////////////////////////////
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
/*@C
TaoVecGetSubVec - Gets a subvector using the IS
Input Parameters:
+ vfull - the full matrix
. is - the index set for the subvector
. reduced_type - the method TAO is using for subsetting (TAO_SUBSET_SUBVEC, TAO_SUBSET_MASK, TAO_SUBSET_MATRIXFREE)
- maskvalue - the value to set the unused vector elements to (for TAO_SUBSET_MASK or TAO_SUBSET_MATRIXFREE)
Output Parameters:
. vreduced - the subvector
Notes:
maskvalue should usually be 0.0, unless a pointwise divide will be used.
@*/
PetscErrorCode TaoVecGetSubVec(Vec vfull, IS is, TaoSubsetType reduced_type, PetscReal maskvalue, Vec *vreduced)
{
PetscErrorCode ierr;
PetscInt nfull,nreduced,nreduced_local,rlow,rhigh,flow,fhigh;
PetscInt i,nlocal;
PetscReal *fv,*rv;
const PetscInt *s;
IS ident;
VecType vtype;
VecScatter scatter;
MPI_Comm comm;
PetscFunctionBegin;
PetscValidHeaderSpecific(vfull,VEC_CLASSID,1);
PetscValidHeaderSpecific(is,IS_CLASSID,2);
ierr = VecGetSize(vfull, &nfull);CHKERRQ(ierr);
ierr = ISGetSize(is, &nreduced);CHKERRQ(ierr);
if (nreduced == nfull) {
ierr = VecDestroy(vreduced);CHKERRQ(ierr);
ierr = VecDuplicate(vfull,vreduced);CHKERRQ(ierr);
ierr = VecCopy(vfull,*vreduced);CHKERRQ(ierr);
} else {
switch (reduced_type) {
case TAO_SUBSET_SUBVEC:
ierr = VecGetType(vfull,&vtype);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(vfull,&flow,&fhigh);CHKERRQ(ierr);
ierr = ISGetLocalSize(is,&nreduced_local);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)vfull,&comm);CHKERRQ(ierr);
if (*vreduced) {
ierr = VecDestroy(vreduced);CHKERRQ(ierr);
}
ierr = VecCreate(comm,vreduced);CHKERRQ(ierr);
ierr = VecSetType(*vreduced,vtype);CHKERRQ(ierr);
ierr = VecSetSizes(*vreduced,nreduced_local,nreduced);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(*vreduced,&rlow,&rhigh);CHKERRQ(ierr);
ierr = ISCreateStride(comm,nreduced_local,rlow,1,&ident);CHKERRQ(ierr);
ierr = VecScatterCreate(vfull,is,*vreduced,ident,&scatter);CHKERRQ(ierr);
ierr = VecScatterBegin(scatter,vfull,*vreduced,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(scatter,vfull,*vreduced,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&scatter);CHKERRQ(ierr);
ierr = ISDestroy(&ident);CHKERRQ(ierr);
break;
case TAO_SUBSET_MASK:
case TAO_SUBSET_MATRIXFREE:
/* vr[i] = vf[i] if i in is
vr[i] = 0 otherwise */
if (!*vreduced) {
ierr = VecDuplicate(vfull,vreduced);CHKERRQ(ierr);
}
ierr = VecSet(*vreduced,maskvalue);CHKERRQ(ierr);
ierr = ISGetLocalSize(is,&nlocal);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(vfull,&flow,&fhigh);CHKERRQ(ierr);
ierr = VecGetArray(vfull,&fv);CHKERRQ(ierr);
ierr = VecGetArray(*vreduced,&rv);CHKERRQ(ierr);
ierr = ISGetIndices(is,&s);CHKERRQ(ierr);
if (nlocal > (fhigh-flow)) SETERRQ2(PETSC_COMM_WORLD,1,"IS local size %d > Vec local size %d",nlocal,fhigh-flow);
for (i=0;i<nlocal;i++) {
rv[s[i]-flow] = fv[s[i]-flow];
}
ierr = ISRestoreIndices(is,&s);CHKERRQ(ierr);
ierr = VecRestoreArray(vfull,&fv);CHKERRQ(ierr);
ierr = VecRestoreArray(*vreduced,&rv);CHKERRQ(ierr);
break;
}
}
PetscFunctionReturn(0);
}
uses block index sets\n\n";
/* 'mpiexec -n 3 ./ex2 -vecscatter_type mpi3node' might give incorrect solution due to multiple cores write to the same variable */
#include <petscvec.h>
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt bs=1,n=5,i,low;
PetscInt ix0[3] = {5,7,9},iy0[3] = {1,2,4},ix1[3] = {2,3,4},iy1[3] = {0,1,3};
PetscMPIInt size,rank;
PetscScalar *array;
Vec x,y;
IS isx,isy;
VecScatter ctx;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (size <2) SETERRQ(PETSC_COMM_SELF,1,"Must run more than one processor");
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
n = bs*n;
/* Create vector x over shared memory */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetType(x,VECNODE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x,&low,NULL);CHKERRQ(ierr);
ierr = VecGetArray(x,&array);CHKERRQ(ierr);
for (i=0; i<n; i++) {
array[i] = (PetscScalar)(i + low);
}
ierr = VecRestoreArray(x,&array);CHKERRQ(ierr);
/* Create a sequential vector y */
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&y);CHKERRQ(ierr);
ierr = VecSet(y,0.0);CHKERRQ(ierr);
/* Create two index sets */
if (!rank) {
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix0,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy0,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
} else {
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,ix1,PETSC_COPY_VALUES,&isx);CHKERRQ(ierr);
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,3,iy1,PETSC_COPY_VALUES,&isy);CHKERRQ(ierr);
}
if (rank == 10) {
ierr = PetscPrintf(PETSC_COMM_SELF,"\n[%d] isx:\n",rank);CHKERRQ(ierr);
ierr = ISView(isx,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = VecScatterCreateWithData(x,isx,y,isy,&ctx);CHKERRQ(ierr);
ierr = VecScatterSetFromOptions(ctx);CHKERRQ(ierr);
/* Test forward vecscatter */
ierr = VecScatterBegin(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,ADD_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
if (rank == 0) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%d] y:\n",rank);CHKERRQ(ierr);
ierr = VecView(y,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
/* Test reverse vecscatter */
ierr = VecScale(y,-1.0);CHKERRQ(ierr);
if (rank) {
ierr = VecScale(y,1.0/(size - 1));CHKERRQ(ierr);
}
ierr = VecScatterBegin(ctx,y,x,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,y,x,ADD_VALUES,SCATTER_REVERSE);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Free spaces */
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = ISDestroy(&isx);CHKERRQ(ierr);
ierr = ISDestroy(&isy);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt n = 5,i,*blks,bs = 1,m = 2;
PetscScalar value;
Vec x,y;
IS is1,is2;
VecScatter ctx = 0;
PetscViewer sviewer;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* create two vectors */
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,PETSC_DECIDE,size*bs*n);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
/* create two index sets */
if (rank < size-1) m = n + 2;
else m = n;
ierr = PetscMalloc1(m,&blks);CHKERRQ(ierr);
blks[0] = n*rank;
for (i=1; i<m; i++) blks[i] = blks[i-1] + 1;
ierr = ISCreateBlock(PETSC_COMM_SELF,bs,m,blks,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = PetscFree(blks);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,bs*m,&y);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,bs*m,0,1,&is2);CHKERRQ(ierr);
/* each processor inserts the entire vector */
/* this is redundant but tests assembly */
for (i=0; i<bs*n*size; i++) {
value = (PetscScalar) i;
ierr = VecSetValues(x,1,&i,&value,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,y,is2,&ctx);CHKERRQ(ierr);
ierr = VecScatterBegin(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(ctx,x,y,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(PETSC_VIEWER_STDOUT_WORLD,"----\n");CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = VecView(y,sviewer);CHKERRQ(ierr); fflush(stdout);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&ctx);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}