/* Compose an IS with an ISLocalToGlobalMapping to map from IS source indices to global indices */
static PetscErrorCode ISL2GCompose(IS is,ISLocalToGlobalMapping ltog,ISLocalToGlobalMapping *cltog)
{
PetscErrorCode ierr;
const PetscInt *idx;
PetscInt m,*idxm;
PetscFunctionBegin;
PetscValidHeaderSpecific(is,IS_CLASSID,1);
PetscValidHeaderSpecific(ltog,IS_LTOGM_CLASSID,2);
PetscValidPointer(cltog,3);
ierr = ISGetLocalSize(is,&m);CHKERRQ(ierr);
ierr = ISGetIndices(is,&idx);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
{
PetscInt i;
for (i=0; i<m; i++) {
if (idx[i] < 0 || ltog->n <= idx[i]) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"is[%D] = %D is not in the local range [0:%D]",i,idx[i],ltog->n);
}
}
#endif
ierr = PetscMalloc(m*sizeof(PetscInt),&idxm);CHKERRQ(ierr);
if (ltog) {
ierr = ISLocalToGlobalMappingApply(ltog,m,idx,idxm);CHKERRQ(ierr);
} else {
ierr = PetscMemcpy(idxm,idx,m*sizeof(PetscInt));CHKERRQ(ierr);
}
ierr = ISLocalToGlobalMappingCreate(((PetscObject)is)->comm,m,idxm,PETSC_OWN_POINTER,cltog);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&idx);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatPartitioningHierarchical_AssembleSubdomain(Mat adj,IS destination,Mat *sadj, ISLocalToGlobalMapping *mapping)
{
IS irows,icols;
PetscInt irows_ln;
PetscMPIInt rank;
const PetscInt *irows_indices;
MPI_Comm comm;
PetscErrorCode ierr;
PetscFunctionBegin;
/*get comm*/
ierr = PetscObjectGetComm((PetscObject)adj,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
/*get rows from remote and local */
ierr = ISBuildTwoSided(destination,NULL,&irows);CHKERRQ(ierr);
ierr = ISDuplicate(irows,&icols);CHKERRQ(ierr);
/*get rows information */
ierr = ISGetLocalSize(irows,&irows_ln);CHKERRQ(ierr);
ierr = ISGetIndices(irows,&irows_indices);CHKERRQ(ierr);
/* create a mapping from local to global */
ierr = ISLocalToGlobalMappingCreate(comm,1,irows_ln,irows_indices,PETSC_COPY_VALUES,mapping);CHKERRQ(ierr);
ierr = ISRestoreIndices(irows,&irows_indices);CHKERRQ(ierr);
/* extract a submatrix*/
ierr = MatGetSubMatrices(adj,1,&irows,&icols,MAT_INITIAL_MATRIX,&sadj);CHKERRQ(ierr);
ierr = ISDestroy(&irows);CHKERRQ(ierr);
ierr = ISDestroy(&icols);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscInt nlocal,local[5],nneigh,*neigh,**ineigh,*numneigh;
ISLocalToGlobalMapping mapping;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 3) SETERRQ(PETSC_COMM_SELF,1,"Must run with three processors");
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
if (!rank) {
nlocal = 4; local[0] = 0; local[1] = 3; local[2] = 2; local[3] = 1;
} else if (rank == 1) {
nlocal = 4; local[0] = 3; local[1] = 5; local[2] = 4; local[3] = 2;
} else {
nlocal = 4; local[0] = 7; local[1] = 6; local[2] = 5; local[3] = 3;
}
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,nlocal,local,PETSC_COPY_VALUES,&mapping);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetInfo(mapping,&nneigh,&neigh,&numneigh,&ineigh);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreInfo(mapping,&nneigh,&neigh,&numneigh,&ineigh);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&mapping);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@
VecMPISetGhost - Sets the ghost points for an MPI ghost vector
Collective on Vec
Input Parameters:
+ vv - the MPI vector
. nghost - number of local ghost points
- ghosts - global indices of ghost points, these do not need to be in increasing order (sorted)
Notes:
Use VecGhostGetLocalForm() to access the local, ghosted representation
of the vector.
This also automatically sets the ISLocalToGlobalMapping() for this vector.
You must call this AFTER you have set the type of the vector (with VecSetType()) and the size (with VecSetSizes()).
Level: advanced
Concepts: vectors^ghosted
.seealso: VecCreateSeq(), VecCreate(), VecDuplicate(), VecDuplicateVecs(), VecCreateMPI(),
VecGhostGetLocalForm(), VecGhostRestoreLocalForm(), VecGhostUpdateBegin(),
VecCreateGhostWithArray(), VecCreateMPIWithArray(), VecGhostUpdateEnd(),
VecCreateGhostBlock(), VecCreateGhostBlockWithArray()
@*/
PetscErrorCode VecMPISetGhost(Vec vv,PetscInt nghost,const PetscInt ghosts[])
{
PetscErrorCode ierr;
PetscBool flg;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)vv,VECMPI,&flg);CHKERRQ(ierr);
/* if already fully existant VECMPI then basically destroy it and rebuild with ghosting */
if (flg) {
PetscInt n,N;
Vec_MPI *w;
PetscScalar *larray;
IS from,to;
ISLocalToGlobalMapping ltog;
PetscInt rstart,i,*indices;
MPI_Comm comm = ((PetscObject)vv)->comm;
n = vv->map->n;
N = vv->map->N;
ierr = (*vv->ops->destroy)(vv);CHKERRQ(ierr);
ierr = VecSetSizes(vv,n,N);CHKERRQ(ierr);
ierr = VecCreate_MPI_Private(vv,PETSC_TRUE,nghost,PETSC_NULL);CHKERRQ(ierr);
w = (Vec_MPI *)(vv)->data;
/* Create local representation */
ierr = VecGetArray(vv,&larray);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,1,n+nghost,larray,&w->localrep);CHKERRQ(ierr);
ierr = PetscLogObjectParent(vv,w->localrep);CHKERRQ(ierr);
ierr = VecRestoreArray(vv,&larray);CHKERRQ(ierr);
/*
Create scatter context for scattering (updating) ghost values
*/
ierr = ISCreateGeneral(comm,nghost,ghosts,PETSC_COPY_VALUES,&from);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,nghost,n,1,&to);CHKERRQ(ierr);
ierr = VecScatterCreate(vv,from,w->localrep,to,&w->localupdate);CHKERRQ(ierr);
ierr = PetscLogObjectParent(vv,w->localupdate);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
/* set local to global mapping for ghosted vector */
ierr = PetscMalloc((n+nghost)*sizeof(PetscInt),&indices);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(vv,&rstart,PETSC_NULL);CHKERRQ(ierr);
for (i=0; i<n; i++) {
indices[i] = rstart + i;
}
for (i=0; i<nghost; i++) {
indices[n+i] = ghosts[i];
}
ierr = ISLocalToGlobalMappingCreate(comm,n+nghost,indices,PETSC_OWN_POINTER,<og);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(vv,ltog);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(<og);CHKERRQ(ierr);
} else if (vv->ops->create == VecCreate_MPI) SETERRQ(((PetscObject)vv)->comm,PETSC_ERR_ARG_WRONGSTATE,"Must set local or global size before setting ghosting");
else if (!((PetscObject)vv)->type_name) SETERRQ(((PetscObject)vv)->comm,PETSC_ERR_ARG_WRONGSTATE,"Must set type to VECMPI before ghosting");
PetscFunctionReturn(0);
}
/*@C
VecCreateGhostBlockWithArray - Creates a parallel vector with ghost padding on each processor;
the caller allocates the array space. Indices in the ghost region are based on blocks.
Collective on MPI_Comm
Input Parameters:
+ comm - the MPI communicator to use
. bs - block size
. n - local vector length
. N - global vector length (or PETSC_DECIDE to have calculated if n is given)
. nghost - number of local ghost blocks
. ghosts - global indices of ghost blocks (or PETSC_NULL if not needed), counts are by block not by index, these do not need to be in increasing order (sorted)
- array - the space to store the vector values (as long as n + nghost*bs)
Output Parameter:
. vv - the global vector representation (without ghost points as part of vector)
Notes:
Use VecGhostGetLocalForm() to access the local, ghosted representation
of the vector.
n is the local vector size (total local size not the number of blocks) while nghost
is the number of blocks in the ghost portion, i.e. the number of elements in the ghost
portion is bs*nghost
Level: advanced
Concepts: vectors^creating ghosted
Concepts: vectors^creating with array
.seealso: VecCreate(), VecGhostGetLocalForm(), VecGhostRestoreLocalForm(),
VecCreateGhost(), VecCreateSeqWithArray(), VecCreateMPIWithArray(),
VecCreateGhostWithArray(), VecCreateGhostBlock()
@*/
PetscErrorCode VecCreateGhostBlockWithArray(MPI_Comm comm,PetscInt bs,PetscInt n,PetscInt N,PetscInt nghost,const PetscInt ghosts[],const PetscScalar array[],Vec *vv)
{
PetscErrorCode ierr;
Vec_MPI *w;
PetscScalar *larray;
IS from,to;
ISLocalToGlobalMapping ltog;
PetscInt rstart,i,nb,*indices;
PetscFunctionBegin;
*vv = 0;
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Must set local size");
if (nghost == PETSC_DECIDE) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Must set local ghost size");
if (nghost < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Ghost length must be >= 0");
if (n % bs) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Local size must be a multiple of block size");
ierr = PetscSplitOwnership(comm,&n,&N);CHKERRQ(ierr);
/* Create global representation */
ierr = VecCreate(comm,vv);CHKERRQ(ierr);
ierr = VecSetSizes(*vv,n,N);CHKERRQ(ierr);
ierr = VecSetBlockSize(*vv,bs);CHKERRQ(ierr);
ierr = VecCreate_MPI_Private(*vv,PETSC_TRUE,nghost*bs,array);CHKERRQ(ierr);
w = (Vec_MPI *)(*vv)->data;
/* Create local representation */
ierr = VecGetArray(*vv,&larray);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,bs,n+bs*nghost,larray,&w->localrep);CHKERRQ(ierr);
ierr = PetscLogObjectParent(*vv,w->localrep);CHKERRQ(ierr);
ierr = VecRestoreArray(*vv,&larray);CHKERRQ(ierr);
/*
Create scatter context for scattering (updating) ghost values
*/
ierr = ISCreateBlock(comm,bs,nghost,ghosts,PETSC_COPY_VALUES,&from);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,bs*nghost,n,1,&to);CHKERRQ(ierr);
ierr = VecScatterCreate(*vv,from,w->localrep,to,&w->localupdate);CHKERRQ(ierr);
ierr = PetscLogObjectParent(*vv,w->localupdate);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
/* set local to global mapping for ghosted vector */
nb = n/bs;
ierr = PetscMalloc((nb+nghost)*sizeof(PetscInt),&indices);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(*vv,&rstart,PETSC_NULL);CHKERRQ(ierr);
for (i=0; i<nb; i++) {
indices[i] = rstart + i*bs;
}
for (i=0; i<nghost; i++) {
indices[nb+i] = ghosts[i];
}
ierr = ISLocalToGlobalMappingCreate(comm,nb+nghost,indices,PETSC_OWN_POINTER,<og);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMappingBlock(*vv,ltog);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(<og);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMCompositeGetISLocalToGlobalMappings - gets an ISLocalToGlobalMapping for each DM in the DMComposite, maps to the composite global space
Collective on DM
Input Parameter:
. dm - the packer object
Output Parameters:
. ltogs - the individual mappings for each packed vector. Note that this includes
all the ghost points that individual ghosted DMDA's may have.
Level: advanced
Notes:
Each entry of ltogs should be destroyed with ISLocalToGlobalMappingDestroy(), the ltogs array should be freed with PetscFree().
.seealso DMDestroy(), DMCompositeAddDM(), DMCreateGlobalVector(),
DMCompositeGather(), DMCompositeCreate(), DMCompositeGetAccess(), DMCompositeScatter(),
DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),DMCompositeGetEntries()
@*/
PetscErrorCode DMCompositeGetISLocalToGlobalMappings(DM dm,ISLocalToGlobalMapping **ltogs)
{
PetscErrorCode ierr;
PetscInt i,*idx,n,cnt;
struct DMCompositeLink *next;
PetscMPIInt rank;
DM_Composite *com = (DM_Composite*)dm->data;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm,DM_CLASSID,1);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = PetscMalloc((com->nDM)*sizeof(ISLocalToGlobalMapping),ltogs);CHKERRQ(ierr);
next = com->next;
ierr = MPI_Comm_rank(((PetscObject)dm)->comm,&rank);CHKERRQ(ierr);
/* loop over packed objects, handling one at at time */
cnt = 0;
while (next) {
ISLocalToGlobalMapping ltog;
PetscMPIInt size;
const PetscInt *suboff,*indices;
Vec global;
/* Get sub-DM global indices for each local dof */
ierr = DMGetLocalToGlobalMapping(next->dm,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltog,&n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog,&indices);CHKERRQ(ierr);
ierr = PetscMalloc(n*sizeof(PetscInt),&idx);CHKERRQ(ierr);
/* Get the offsets for the sub-DM global vector */
ierr = DMGetGlobalVector(next->dm,&global);CHKERRQ(ierr);
ierr = VecGetOwnershipRanges(global,&suboff);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)global)->comm,&size);CHKERRQ(ierr);
/* Shift the sub-DM definition of the global space to the composite global space */
for (i=0; i<n; i++) {
PetscInt subi = indices[i],lo = 0,hi = size,t;
/* Binary search to find which rank owns subi */
while (hi-lo > 1) {
t = lo + (hi-lo)/2;
if (suboff[t] > subi) hi = t;
else lo = t;
}
idx[i] = subi - suboff[lo] + next->grstarts[lo];
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltog,&indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm,n,idx,PETSC_OWN_POINTER,&(*ltogs)[cnt]);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(next->dm,&global);CHKERRQ(ierr);
next = next->next;
cnt++;
}
PetscFunctionReturn(0);
}
/* Compose an IS with an ISLocalToGlobalMapping to map from IS source indices to global indices */
static PetscErrorCode ISL2GCompose(IS is,ISLocalToGlobalMapping ltog,ISLocalToGlobalMapping *cltog)
{
PetscErrorCode ierr;
const PetscInt *idx;
PetscInt m,*idxm;
PetscBool isblock;
PetscFunctionBegin;
PetscValidHeaderSpecific(is,IS_CLASSID,1);
PetscValidHeaderSpecific(ltog,IS_LTOGM_CLASSID,2);
PetscValidPointer(cltog,3);
ierr = PetscObjectTypeCompare((PetscObject)is,ISBLOCK,&isblock);CHKERRQ(ierr);
if (isblock) {
PetscInt bs,lbs;
ierr = ISGetBlockSize(is,&bs);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetBlockSize(ltog,&lbs);CHKERRQ(ierr);
if (bs == lbs) {
ierr = ISGetLocalSize(is,&m);CHKERRQ(ierr);
m = m/bs;
ierr = ISBlockGetIndices(is,&idx);CHKERRQ(ierr);
ierr = PetscMalloc1(m,&idxm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingApplyBlock(ltog,m,idx,idxm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)is),bs,m,idxm,PETSC_OWN_POINTER,cltog);CHKERRQ(ierr);
ierr = ISBlockRestoreIndices(is,&idx);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
}
ierr = ISGetLocalSize(is,&m);CHKERRQ(ierr);
ierr = ISGetIndices(is,&idx);CHKERRQ(ierr);
ierr = PetscMalloc1(m,&idxm);CHKERRQ(ierr);
if (ltog) {
ierr = ISLocalToGlobalMappingApply(ltog,m,idx,idxm);CHKERRQ(ierr);
} else {
ierr = PetscMemcpy(idxm,idx,m*sizeof(PetscInt));CHKERRQ(ierr);
}
ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)is),1,m,idxm,PETSC_OWN_POINTER,cltog);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&idx);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt indices[] = {0,1,2,3,-1,-1,-1,-1,4,5,6,7};
PetscInt indices2[] = {0,1,2,3,4,5,-1,-1,-1,-1,-1,-1,6,7,8,9,10,11};
ISLocalToGlobalMapping map;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,1,12,indices,PETSC_COPY_VALUES,&map);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,2);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,4);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,2);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,1);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&map);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,1,18,indices2,PETSC_COPY_VALUES,&map);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,3);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,6);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,3);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,1);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&map);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,5,2,indices2,PETSC_COPY_VALUES,&map);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingSetBlockSize(map,2);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(map,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&map);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
static PetscErrorCode DMSetUp_Redundant(DM dm)
{
PetscErrorCode ierr;
DM_Redundant *red = (DM_Redundant*)dm->data;
PetscInt i,*globals;
PetscFunctionBegin;
ierr = PetscMalloc1(red->N,&globals);CHKERRQ(ierr);
for (i=0; i<red->N; i++) globals[i] = i;
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,red->N,globals,PETSC_OWN_POINTER,&dm->ltogmap);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)dm->ltogmap);CHKERRQ(ierr);
dm->ltogmapb = dm->ltogmap;
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscInt i,n = 4,indices[] = {0,3,9,12},m = 2,input[] = {0,2};
PetscInt output[2],inglobals[13],outlocals[13];
ISLocalToGlobalMapping mapping;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);CHKERRQ(ierr);
/*
Create a local to global mapping. Each processor independently
creates a mapping
*/
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,n,indices,PETSC_COPY_VALUES,&mapping);CHKERRQ(ierr);
/*
Map a set of local indices to their global values
*/
ierr = ISLocalToGlobalMappingApply(mapping,m,input,output);CHKERRQ(ierr);
ierr = PetscIntView(m,output,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Map some global indices to local, retaining the ones without a local index by -1
*/
for (i=0; i<13; i++) {
inglobals[i] = i;
}
ierr = ISGlobalToLocalMappingApply(mapping,IS_GTOLM_MASK,13,inglobals,PETSC_NULL,outlocals);CHKERRQ(ierr);
ierr = PetscIntView(13,outlocals,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Map some global indices to local, dropping the ones without a local index.
*/
ierr = ISGlobalToLocalMappingApply(mapping,IS_GTOLM_DROP,13,inglobals,&m,outlocals);CHKERRQ(ierr);
ierr = PetscIntView(m,outlocals,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
/*
Free the space used by the local to global mapping
*/
ierr = ISLocalToGlobalMappingDestroy(&mapping);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
Decomposition::Decomposition(PetscInt istart, PetscInt iend, const Grid::FaceList& faces):
cellStart(istart),
cellEnd(iend)
{
std::set<PetscInt> ghosts;
for (auto f: faces)
if ((f.neighbour < istart || iend <= f.neighbour) && f.neighbour >= 0)
ghosts.insert(f.neighbour);
std::vector<PetscInt> glob(cellEnd - cellStart + ghosts.size());
int id=0;
for (int i=cellStart; i<cellEnd; i++) glob[id++] = i;
for (auto g: ghosts) glob[id++] = g;
ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD, 1, glob.size(), glob.data(),
PETSC_COPY_VALUES, &locToGlobMap);
}
/*@
ISLocalToGlobalMappingCreateIS - Creates a mapping between a local (0 to n)
ordering and a global parallel ordering.
Not collective
Input Parameter:
. is - index set containing the global numbers for each local
Output Parameter:
. mapping - new mapping data structure
Level: advanced
Concepts: mapping^local to global
.seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate()
@*/
PetscErrorCode PETSCVEC_DLLEXPORT ISLocalToGlobalMappingCreateIS(IS is,ISLocalToGlobalMapping *mapping)
{
PetscErrorCode ierr;
PetscInt n;
const PetscInt *indices;
MPI_Comm comm;
PetscFunctionBegin;
PetscValidHeaderSpecific(is,IS_COOKIE,1);
PetscValidPointer(mapping,2);
ierr = PetscObjectGetComm((PetscObject)is,&comm);CHKERRQ(ierr);
ierr = ISGetLocalSize(is,&n);CHKERRQ(ierr);
ierr = ISGetIndices(is,&indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(comm,n,indices,mapping);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&indices);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ComputeMapping(DomainData dd,ISLocalToGlobalMapping *isg2lmap)
{
PetscErrorCode ierr;
DM da;
AO ao;
DMBoundaryType bx = DM_BOUNDARY_NONE,by = DM_BOUNDARY_NONE, bz = DM_BOUNDARY_NONE;
DMDAStencilType stype = DMDA_STENCIL_BOX;
ISLocalToGlobalMapping temp_isg2lmap;
PetscInt i,j,k,ig,jg,kg,lindex,gindex,localsize;
PetscInt *global_indices;
PetscFunctionBeginUser;
/* Not an efficient mapping: this function computes a very simple lexicographic mapping
just to illustrate the creation of a MATIS object */
localsize = dd.xm_l*dd.ym_l*dd.zm_l;
ierr = PetscMalloc1(localsize,&global_indices);CHKERRQ(ierr);
for (k=0; k<dd.zm_l; k++) {
kg=dd.startz+k;
for (j=0; j<dd.ym_l; j++) {
jg=dd.starty+j;
for (i=0; i<dd.xm_l; i++) {
ig =dd.startx+i;
lindex =k*dd.xm_l*dd.ym_l+j*dd.xm_l+i;
gindex =kg*dd.xm*dd.ym+jg*dd.xm+ig;
global_indices[lindex]=gindex;
}
}
}
if (dd.dim==3) {
ierr = DMDACreate3d(dd.gcomm,bx,by,bz,stype,dd.xm,dd.ym,dd.zm,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
} else if (dd.dim==2) {
ierr = DMDACreate2d(dd.gcomm,bx,by,stype,dd.xm,dd.ym,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr);
} else {
ierr = DMDACreate1d(dd.gcomm,bx,dd.xm,1,1,NULL,&da);CHKERRQ(ierr);
}
ierr = DMDASetAOType(da,AOMEMORYSCALABLE);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = AOApplicationToPetsc(ao,dd.xm_l*dd.ym_l*dd.zm_l,global_indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(dd.gcomm,1,localsize,global_indices,PETSC_OWN_POINTER,&temp_isg2lmap);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
*isg2lmap = temp_isg2lmap;
PetscFunctionReturn(0);
}
/* Compose an IS with an ISLocalToGlobalMapping to map from IS source indices to global indices */
static PetscErrorCode ISL2GCompose(IS is,ISLocalToGlobalMapping ltog,ISLocalToGlobalMapping *cltog)
{
PetscErrorCode ierr;
const PetscInt *idx;
PetscInt m,*idxm;
PetscFunctionBegin;
PetscValidHeaderSpecific(is,IS_CLASSID,1);
PetscValidHeaderSpecific(ltog,IS_LTOGM_CLASSID,2);
PetscValidPointer(cltog,3);
ierr = ISGetLocalSize(is,&m);CHKERRQ(ierr);
ierr = ISGetIndices(is,&idx);CHKERRQ(ierr);
ierr = PetscMalloc1(m,&idxm);CHKERRQ(ierr);
if (ltog) {
ierr = ISLocalToGlobalMappingApply(ltog,m,idx,idxm);CHKERRQ(ierr);
} else {
ierr = PetscMemcpy(idxm,idx,m*sizeof(PetscInt));CHKERRQ(ierr);
}
ierr = ISLocalToGlobalMappingCreate(PetscObjectComm((PetscObject)is),m,idxm,PETSC_OWN_POINTER,cltog);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&idx);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
ISLocalToGlobalMappingBlock - Creates a blocked index version of an
ISLocalToGlobalMapping that is appropriate for MatSetLocalToGlobalMappingBlock()
and VecSetLocalToGlobalMappingBlock().
Not Collective, but communicator may have more than one process
Input Parameters:
+ inmap - original point-wise mapping
- bs - block size
Output Parameter:
. outmap - block based mapping; the indices are relative to BLOCKS, not individual vector or matrix entries.
Level: advanced
Concepts: mapping^local to global
.seealso: ISLocalToGlobalMappingDestroy(), ISLocalToGlobalMappingCreate(), ISLocalToGlobalMappingCreateIS()
@*/
PetscErrorCode PETSCVEC_DLLEXPORT ISLocalToGlobalMappingBlock(ISLocalToGlobalMapping inmap,PetscInt bs,ISLocalToGlobalMapping *outmap)
{
PetscErrorCode ierr;
PetscInt *ii,i,n;
PetscFunctionBegin;
PetscValidHeaderSpecific(inmap,IS_LTOGM_COOKIE,1);
PetscValidPointer(outmap,1);
if (bs > 1) {
n = inmap->n/bs;
if (n*bs != inmap->n) SETERRQ(PETSC_ERR_ARG_INCOMP,"Pointwise mapping length is not divisible by block size");
ierr = PetscMalloc(n*sizeof(PetscInt),&ii);CHKERRQ(ierr);
for (i=0; i<n; i++) {
ii[i] = inmap->indices[bs*i]/bs;
}
ierr = ISLocalToGlobalMappingCreate(((PetscObject)inmap)->comm,n,ii,outmap);CHKERRQ(ierr);
ierr = PetscFree(ii);CHKERRQ(ierr);
} else {
ierr = PetscObjectReference((PetscObject)inmap);CHKERRQ(ierr);
*outmap = inmap;
}
PetscFunctionReturn(0);
}
PetscErrorCode DMSetUp_DA_1D(DM da)
{
DM_DA *dd = (DM_DA*)da->data;
const PetscInt M = dd->M;
const PetscInt dof = dd->w;
const PetscInt s = dd->s;
const PetscInt sDist = s; /* stencil distance in points */
const PetscInt *lx = dd->lx;
DMBoundaryType bx = dd->bx;
MPI_Comm comm;
Vec local, global;
VecScatter gtol;
IS to, from;
PetscBool flg1 = PETSC_FALSE, flg2 = PETSC_FALSE;
PetscMPIInt rank, size;
PetscInt i,*idx,nn,left,xs,xe,x,Xs,Xe,start,m,IXs,IXe;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject) da, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
dd->p = 1;
dd->n = 1;
dd->m = size;
m = dd->m;
if (s > 0) {
/* if not communicating data then should be ok to have nothing on some processes */
if (M < m) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"More processes than data points! %D %D",m,M);
if ((M-1) < s && size > 1) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Array is too small for stencil! %D %D",M-1,s);
}
/*
Determine locally owned region
xs is the first local node number, x is the number of local nodes
*/
if (!lx) {
ierr = PetscMalloc1(m, &dd->lx);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-da_partition_blockcomm",&flg1,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL,"-da_partition_nodes_at_end",&flg2,NULL);CHKERRQ(ierr);
if (flg1) { /* Block Comm type Distribution */
xs = rank*M/m;
x = (rank + 1)*M/m - xs;
} else if (flg2) { /* The odd nodes are evenly distributed across last nodes */
x = (M + rank)/m;
if (M/m == x) xs = rank*x;
else xs = rank*(x-1) + (M+rank)%(x*m);
} else { /* The odd nodes are evenly distributed across the first k nodes */
/* Regular PETSc Distribution */
x = M/m + ((M % m) > rank);
if (rank >= (M % m)) xs = (rank * (PetscInt)(M/m) + M % m);
else xs = rank * (PetscInt)(M/m) + rank;
}
ierr = MPI_Allgather(&xs,1,MPIU_INT,dd->lx,1,MPIU_INT,comm);CHKERRQ(ierr);
for (i=0; i<m-1; i++) dd->lx[i] = dd->lx[i+1] - dd->lx[i];
dd->lx[m-1] = M - dd->lx[m-1];
} else {
x = lx[rank];
xs = 0;
for (i=0; i<rank; i++) xs += lx[i];
/* verify that data user provided is consistent */
left = xs;
for (i=rank; i<size; i++) left += lx[i];
if (left != M) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Sum of lx across processors not equal to M %D %D",left,M);
}
/*
check if the scatter requires more than one process neighbor or wraps around
the domain more than once
*/
if ((x < s) & ((M > 1) | (bx == DM_BOUNDARY_PERIODIC))) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Local x-width of domain x %D is smaller than stencil width s %D",x,s);
xe = xs + x;
/* determine ghost region (Xs) and region scattered into (IXs) */
if (xs-sDist > 0) {
Xs = xs - sDist;
IXs = xs - sDist;
} else {
if (bx) Xs = xs - sDist;
else Xs = 0;
IXs = 0;
}
if (xe+sDist <= M) {
Xe = xe + sDist;
IXe = xe + sDist;
} else {
if (bx) Xe = xe + sDist;
else Xe = M;
IXe = M;
}
if (bx == DM_BOUNDARY_PERIODIC || bx == DM_BOUNDARY_MIRROR) {
Xs = xs - sDist;
Xe = xe + sDist;
IXs = xs - sDist;
IXe = xe + sDist;
}
/* allocate the base parallel and sequential vectors */
dd->Nlocal = dof*x;
ierr = VecCreateMPIWithArray(comm,dof,dd->Nlocal,PETSC_DECIDE,NULL,&global);CHKERRQ(ierr);
dd->nlocal = dof*(Xe-Xs);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,dof,dd->nlocal,NULL,&local);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(global,&start,NULL);CHKERRQ(ierr);
/* Create Global to Local Vector Scatter Context */
/* global to local must retrieve ghost points */
ierr = ISCreateStride(comm,dof*(IXe-IXs),dof*(IXs-Xs),1,&to);CHKERRQ(ierr);
ierr = PetscMalloc1(x+2*sDist,&idx);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)da,(x+2*(sDist))*sizeof(PetscInt));CHKERRQ(ierr);
for (i=0; i<IXs-Xs; i++) idx[i] = -1; /* prepend with -1s if needed for ghosted case*/
nn = IXs-Xs;
if (bx == DM_BOUNDARY_PERIODIC) { /* Handle all cases with periodic first */
for (i=0; i<sDist; i++) { /* Left ghost points */
if ((xs-sDist+i)>=0) idx[nn++] = xs-sDist+i;
else idx[nn++] = M+(xs-sDist+i);
}
for (i=0; i<x; i++) idx [nn++] = xs + i; /* Non-ghost points */
for (i=0; i<sDist; i++) { /* Right ghost points */
if ((xe+i)<M) idx [nn++] = xe+i;
else idx [nn++] = (xe+i) - M;
}
} else if (bx == DM_BOUNDARY_MIRROR) { /* Handle all cases with periodic first */
for (i=0; i<(sDist); i++) { /* Left ghost points */
if ((xs-sDist+i)>=0) idx[nn++] = xs-sDist+i;
else idx[nn++] = sDist - i;
}
for (i=0; i<x; i++) idx [nn++] = xs + i; /* Non-ghost points */
for (i=0; i<(sDist); i++) { /* Right ghost points */
if ((xe+i)<M) idx[nn++] = xe+i;
else idx[nn++] = M - (i + 1);
}
} else { /* Now do all cases with no periodicity */
if (0 <= xs-sDist) {
for (i=0; i<sDist; i++) idx[nn++] = xs - sDist + i;
} else {
for (i=0; i<xs; i++) idx[nn++] = i;
}
for (i=0; i<x; i++) idx [nn++] = xs + i;
if ((xe+sDist)<=M) {
for (i=0; i<sDist; i++) idx[nn++]=xe+i;
} else {
for (i=xe; i<M; i++) idx[nn++]=i;
}
}
ierr = ISCreateBlock(comm,dof,nn-IXs+Xs,&idx[IXs-Xs],PETSC_USE_POINTER,&from);CHKERRQ(ierr);
ierr = VecScatterCreate(global,from,local,to,>ol);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)da,(PetscObject)gtol);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
dd->xs = dof*xs; dd->xe = dof*xe; dd->ys = 0; dd->ye = 1; dd->zs = 0; dd->ze = 1;
dd->Xs = dof*Xs; dd->Xe = dof*Xe; dd->Ys = 0; dd->Ye = 1; dd->Zs = 0; dd->Ze = 1;
dd->gtol = gtol;
dd->base = dof*xs;
da->ops->view = DMView_DA_1d;
/*
Set the local to global ordering in the global vector, this allows use
of VecSetValuesLocal().
*/
for (i=0; i<Xe-IXe; i++) idx[nn++] = -1; /* pad with -1s if needed for ghosted case*/
ierr = ISLocalToGlobalMappingCreate(comm,dof,nn,idx,PETSC_OWN_POINTER,&da->ltogmap);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)da,(PetscObject)da->ltogmap);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char *argv[])
{
PetscErrorCode ierr;
IS is0a,is0b,is0,is1,isl0a,isl0b,isl0,isl1;
Mat A,Aexplicit;
PetscBool usenest;
PetscMPIInt rank,size;
PetscInt i,j;
PetscInitialize(&argc,&argv,PETSC_NULL,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
{
const PetscInt ix0a[] = {rank*2+0},ix0b[] = {rank*2+1},ix0[] = {rank*3+0,rank*3+1},ix1[] = {rank*3+2};
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,ix0a,PETSC_COPY_VALUES,&is0a);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,ix0b,PETSC_COPY_VALUES,&is0b);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,2,ix0,PETSC_COPY_VALUES,&is0);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_WORLD,1,ix1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
}
{
ierr = ISCreateStride(PETSC_COMM_SELF,6,0,1,&isl0);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,3,0,1,&isl0a);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,3,3,1,&isl0b);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,3,6,1,&isl1);CHKERRQ(ierr);
}
usenest = PETSC_FALSE;
ierr = PetscOptionsGetBool(PETSC_NULL,"-nest",&usenest,PETSC_NULL);CHKERRQ(ierr);
if (usenest) {
ISLocalToGlobalMapping l2g;
const PetscInt l2gind[3] = {(rank-1+size)%size,rank,(rank+1)%size};
Mat B[9];
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,3,l2gind,PETSC_COPY_VALUES,&l2g);CHKERRQ(ierr);
for (i=0; i<9; i++) {
ierr = MatCreateAIJ(PETSC_COMM_WORLD,1,1,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,PETSC_NULL,PETSC_DECIDE,PETSC_NULL,&B[i]);CHKERRQ(ierr);
ierr = MatSetUp(B[i]);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(B[i],l2g,l2g);CHKERRQ(ierr);
}
{
const IS isx[] = {is0a,is0b};
const Mat Bx00[] = {B[0],B[1],B[3],B[4]},Bx01[] = {B[2],B[5]},Bx10[] = {B[6],B[7]};
Mat B00,B01,B10;
ierr = MatCreateNest(PETSC_COMM_WORLD,2,isx,2,isx,Bx00,&B00);CHKERRQ(ierr);
ierr = MatSetUp(B00);CHKERRQ(ierr);
ierr = MatCreateNest(PETSC_COMM_WORLD,2,isx,1,PETSC_NULL,Bx01,&B01);CHKERRQ(ierr);
ierr = MatSetUp(B01);CHKERRQ(ierr);
ierr = MatCreateNest(PETSC_COMM_WORLD,1,PETSC_NULL,2,isx,Bx10,&B10);CHKERRQ(ierr);
ierr = MatSetUp(B10);CHKERRQ(ierr);
{
Mat By[] = {B00,B01,B10,B[8]};
IS isy[] = {is0,is1};
ierr = MatCreateNest(PETSC_COMM_WORLD,2,isy,2,isy,By,&A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
}
ierr = MatDestroy(&B00);CHKERRQ(ierr);
ierr = MatDestroy(&B01);CHKERRQ(ierr);
ierr = MatDestroy(&B10);CHKERRQ(ierr);
}
for (i=0; i<9; i++) {ierr = MatDestroy(&B[i]);CHKERRQ(ierr);}
ierr = ISLocalToGlobalMappingDestroy(&l2g);CHKERRQ(ierr);
} else {
ISLocalToGlobalMapping l2g;
PetscInt l2gind[9];
for (i=0; i<3; i++) for (j=0; j<3; j++) l2gind[3*i+j] = ((rank-1+j+size) % size)*3 + i;
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_WORLD,9,l2gind,PETSC_COPY_VALUES,&l2g);CHKERRQ(ierr);
ierr = MatCreateAIJ(PETSC_COMM_WORLD,3,3,PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,PETSC_NULL,PETSC_DECIDE,PETSC_NULL,&A);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(A,l2g,l2g);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&l2g);CHKERRQ(ierr);
}
{
Mat A00,A11,A0a0a,A0a0b;
ierr = MatGetLocalSubMatrix(A,isl0,isl0,&A00);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(A,isl1,isl1,&A11);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(A00,isl0a,isl0a,&A0a0a);CHKERRQ(ierr);
ierr = MatGetLocalSubMatrix(A00,isl0a,isl0b,&A0a0b);CHKERRQ(ierr);
ierr = MatSetValueLocal(A0a0a,0,0,100*rank+1,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A0a0a,0,1,100*rank+2,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A0a0a,2,2,100*rank+9,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A0a0b,1,1,100*rank+50+5,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A11,0,0,1000*(rank+1)+1,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValueLocal(A11,1,2,1000*(rank+1)+6,ADD_VALUES);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(A00,isl0a,isl0a,&A0a0a);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(A00,isl0a,isl0b,&A0a0b);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(A,isl0,isl0,&A00);CHKERRQ(ierr);
ierr = MatRestoreLocalSubMatrix(A,isl1,isl1,&A11);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatComputeExplicitOperator(A,&Aexplicit);CHKERRQ(ierr);
ierr = MatView(Aexplicit,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&Aexplicit);CHKERRQ(ierr);
ierr = ISDestroy(&is0a);CHKERRQ(ierr);
ierr = ISDestroy(&is0b);CHKERRQ(ierr);
ierr = ISDestroy(&is0);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&isl0a);CHKERRQ(ierr);
ierr = ISDestroy(&isl0b);CHKERRQ(ierr);
ierr = ISDestroy(&isl0);CHKERRQ(ierr);
ierr = ISDestroy(&isl1);CHKERRQ(ierr);
PetscFinalize();
return 0;
}
PetscErrorCode DMSetUp_DA_2D(DM da)
{
DM_DA *dd = (DM_DA*)da->data;
const PetscInt M = dd->M;
const PetscInt N = dd->N;
PetscInt m = dd->m;
PetscInt n = dd->n;
const PetscInt dof = dd->w;
const PetscInt s = dd->s;
DMBoundaryType bx = dd->bx;
DMBoundaryType by = dd->by;
DMDAStencilType stencil_type = dd->stencil_type;
PetscInt *lx = dd->lx;
PetscInt *ly = dd->ly;
MPI_Comm comm;
PetscMPIInt rank,size;
PetscInt xs,xe,ys,ye,x,y,Xs,Xe,Ys,Ye,IXs,IXe,IYs,IYe;
PetscInt up,down,left,right,i,n0,n1,n2,n3,n5,n6,n7,n8,*idx,nn;
PetscInt xbase,*bases,*ldims,j,x_t,y_t,s_t,base,count;
PetscInt s_x,s_y; /* s proportionalized to w */
PetscInt sn0 = 0,sn2 = 0,sn6 = 0,sn8 = 0;
Vec local,global;
VecScatter gtol;
IS to,from;
PetscErrorCode ierr;
PetscFunctionBegin;
if (stencil_type == DMDA_STENCIL_BOX && (bx == DM_BOUNDARY_MIRROR || by == DM_BOUNDARY_MIRROR)) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Mirror boundary and box stencil");
ierr = PetscObjectGetComm((PetscObject)da,&comm);CHKERRQ(ierr);
#if !defined(PETSC_USE_64BIT_INDICES)
if (((Petsc64bitInt) M)*((Petsc64bitInt) N)*((Petsc64bitInt) dof) > (Petsc64bitInt) PETSC_MPI_INT_MAX) SETERRQ3(comm,PETSC_ERR_INT_OVERFLOW,"Mesh of %D by %D by %D (dof) is too large for 32 bit indices",M,N,dof);
#endif
if (dof < 1) SETERRQ1(comm,PETSC_ERR_ARG_OUTOFRANGE,"Must have 1 or more degrees of freedom per node: %D",dof);
if (s < 0) SETERRQ1(comm,PETSC_ERR_ARG_OUTOFRANGE,"Stencil width cannot be negative: %D",s);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
if (m != PETSC_DECIDE) {
if (m < 1) SETERRQ1(comm,PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in X direction: %D",m);
else if (m > size) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in X direction: %D %d",m,size);
}
if (n != PETSC_DECIDE) {
if (n < 1) SETERRQ1(comm,PETSC_ERR_ARG_OUTOFRANGE,"Non-positive number of processors in Y direction: %D",n);
else if (n > size) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"Too many processors in Y direction: %D %d",n,size);
}
if (m == PETSC_DECIDE || n == PETSC_DECIDE) {
if (n != PETSC_DECIDE) {
m = size/n;
} else if (m != PETSC_DECIDE) {
n = size/m;
} else {
/* try for squarish distribution */
m = (PetscInt)(0.5 + PetscSqrtReal(((PetscReal)M)*((PetscReal)size)/((PetscReal)N)));
if (!m) m = 1;
while (m > 0) {
n = size/m;
if (m*n == size) break;
m--;
}
if (M > N && m < n) {PetscInt _m = m; m = n; n = _m;}
}
if (m*n != size) SETERRQ(comm,PETSC_ERR_PLIB,"Unable to create partition, check the size of the communicator and input m and n ");
} else if (m*n != size) SETERRQ(comm,PETSC_ERR_ARG_OUTOFRANGE,"Given Bad partition");
if (M < m) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"Partition in x direction is too fine! %D %D",M,m);
if (N < n) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"Partition in y direction is too fine! %D %D",N,n);
/*
Determine locally owned region
xs is the first local node number, x is the number of local nodes
*/
if (!lx) {
ierr = PetscMalloc1(m, &dd->lx);CHKERRQ(ierr);
lx = dd->lx;
for (i=0; i<m; i++) {
lx[i] = M/m + ((M % m) > i);
}
}
x = lx[rank % m];
xs = 0;
for (i=0; i<(rank % m); i++) {
xs += lx[i];
}
#if defined(PETSC_USE_DEBUG)
left = xs;
for (i=(rank % m); i<m; i++) {
left += lx[i];
}
if (left != M) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Sum of lx across processors not equal to M: %D %D",left,M);
#endif
/*
Determine locally owned region
ys is the first local node number, y is the number of local nodes
*/
if (!ly) {
ierr = PetscMalloc1(n, &dd->ly);CHKERRQ(ierr);
ly = dd->ly;
for (i=0; i<n; i++) {
ly[i] = N/n + ((N % n) > i);
}
}
y = ly[rank/m];
ys = 0;
for (i=0; i<(rank/m); i++) {
ys += ly[i];
}
#if defined(PETSC_USE_DEBUG)
left = ys;
for (i=(rank/m); i<n; i++) {
left += ly[i];
}
if (left != N) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Sum of ly across processors not equal to N: %D %D",left,N);
#endif
/*
check if the scatter requires more than one process neighbor or wraps around
the domain more than once
*/
if ((x < s) && ((m > 1) || (bx == DM_BOUNDARY_PERIODIC))) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Local x-width of domain x %D is smaller than stencil width s %D",x,s);
if ((y < s) && ((n > 1) || (by == DM_BOUNDARY_PERIODIC))) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Local y-width of domain y %D is smaller than stencil width s %D",y,s);
xe = xs + x;
ye = ys + y;
/* determine ghost region (Xs) and region scattered into (IXs) */
if (xs-s > 0) {
Xs = xs - s; IXs = xs - s;
} else {
if (bx) {
Xs = xs - s;
} else {
Xs = 0;
}
IXs = 0;
}
if (xe+s <= M) {
Xe = xe + s; IXe = xe + s;
} else {
if (bx) {
Xs = xs - s; Xe = xe + s;
} else {
Xe = M;
}
IXe = M;
}
if (bx == DM_BOUNDARY_PERIODIC || bx == DM_BOUNDARY_MIRROR) {
IXs = xs - s;
IXe = xe + s;
Xs = xs - s;
Xe = xe + s;
}
if (ys-s > 0) {
Ys = ys - s; IYs = ys - s;
} else {
if (by) {
Ys = ys - s;
} else {
Ys = 0;
}
IYs = 0;
}
if (ye+s <= N) {
Ye = ye + s; IYe = ye + s;
} else {
if (by) {
Ye = ye + s;
} else {
Ye = N;
}
IYe = N;
}
if (by == DM_BOUNDARY_PERIODIC || by == DM_BOUNDARY_MIRROR) {
IYs = ys - s;
IYe = ye + s;
Ys = ys - s;
Ye = ye + s;
}
/* stencil length in each direction */
s_x = s;
s_y = s;
/* determine starting point of each processor */
nn = x*y;
ierr = PetscMalloc2(size+1,&bases,size,&ldims);CHKERRQ(ierr);
ierr = MPI_Allgather(&nn,1,MPIU_INT,ldims,1,MPIU_INT,comm);CHKERRQ(ierr);
bases[0] = 0;
for (i=1; i<=size; i++) {
bases[i] = ldims[i-1];
}
for (i=1; i<=size; i++) {
bases[i] += bases[i-1];
}
base = bases[rank]*dof;
/* allocate the base parallel and sequential vectors */
dd->Nlocal = x*y*dof;
ierr = VecCreateMPIWithArray(comm,dof,dd->Nlocal,PETSC_DECIDE,NULL,&global);CHKERRQ(ierr);
dd->nlocal = (Xe-Xs)*(Ye-Ys)*dof;
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,dof,dd->nlocal,NULL,&local);CHKERRQ(ierr);
/* generate appropriate vector scatters */
/* local to global inserts non-ghost point region into global */
ierr = PetscMalloc1((IXe-IXs)*(IYe-IYs),&idx);CHKERRQ(ierr);
left = xs - Xs; right = left + x;
down = ys - Ys; up = down + y;
count = 0;
for (i=down; i<up; i++) {
for (j=left; j<right; j++) {
idx[count++] = i*(Xe-Xs) + j;
}
}
/* global to local must include ghost points within the domain,
but not ghost points outside the domain that aren't periodic */
if (stencil_type == DMDA_STENCIL_BOX) {
left = IXs - Xs; right = left + (IXe-IXs);
down = IYs - Ys; up = down + (IYe-IYs);
count = 0;
for (i=down; i<up; i++) {
for (j=left; j<right; j++) {
idx[count++] = j + i*(Xe-Xs);
}
}
ierr = ISCreateBlock(comm,dof,count,idx,PETSC_OWN_POINTER,&to);CHKERRQ(ierr);
} else {
/* must drop into cross shape region */
/* ---------|
| top |
|--- ---| up
| middle |
| |
---- ---- down
| bottom |
-----------
Xs xs xe Xe */
left = xs - Xs; right = left + x;
down = ys - Ys; up = down + y;
count = 0;
/* bottom */
for (i=(IYs-Ys); i<down; i++) {
for (j=left; j<right; j++) {
idx[count++] = j + i*(Xe-Xs);
}
}
/* middle */
for (i=down; i<up; i++) {
for (j=(IXs-Xs); j<(IXe-Xs); j++) {
idx[count++] = j + i*(Xe-Xs);
}
}
/* top */
for (i=up; i<up+IYe-ye; i++) {
for (j=left; j<right; j++) {
idx[count++] = j + i*(Xe-Xs);
}
}
ierr = ISCreateBlock(comm,dof,count,idx,PETSC_OWN_POINTER,&to);CHKERRQ(ierr);
}
/* determine who lies on each side of us stored in n6 n7 n8
n3 n5
n0 n1 n2
*/
/* Assume the Non-Periodic Case */
n1 = rank - m;
if (rank % m) {
n0 = n1 - 1;
} else {
n0 = -1;
}
if ((rank+1) % m) {
n2 = n1 + 1;
n5 = rank + 1;
n8 = rank + m + 1; if (n8 >= m*n) n8 = -1;
} else {
n2 = -1; n5 = -1; n8 = -1;
}
if (rank % m) {
n3 = rank - 1;
n6 = n3 + m; if (n6 >= m*n) n6 = -1;
} else {
n3 = -1; n6 = -1;
}
n7 = rank + m; if (n7 >= m*n) n7 = -1;
if (bx == DM_BOUNDARY_PERIODIC && by == DM_BOUNDARY_PERIODIC) {
/* Modify for Periodic Cases */
/* Handle all four corners */
if ((n6 < 0) && (n7 < 0) && (n3 < 0)) n6 = m-1;
if ((n8 < 0) && (n7 < 0) && (n5 < 0)) n8 = 0;
if ((n2 < 0) && (n5 < 0) && (n1 < 0)) n2 = size-m;
if ((n0 < 0) && (n3 < 0) && (n1 < 0)) n0 = size-1;
/* Handle Top and Bottom Sides */
if (n1 < 0) n1 = rank + m * (n-1);
if (n7 < 0) n7 = rank - m * (n-1);
if ((n3 >= 0) && (n0 < 0)) n0 = size - m + rank - 1;
if ((n3 >= 0) && (n6 < 0)) n6 = (rank%m)-1;
if ((n5 >= 0) && (n2 < 0)) n2 = size - m + rank + 1;
if ((n5 >= 0) && (n8 < 0)) n8 = (rank%m)+1;
/* Handle Left and Right Sides */
if (n3 < 0) n3 = rank + (m-1);
if (n5 < 0) n5 = rank - (m-1);
if ((n1 >= 0) && (n0 < 0)) n0 = rank-1;
if ((n1 >= 0) && (n2 < 0)) n2 = rank-2*m+1;
if ((n7 >= 0) && (n6 < 0)) n6 = rank+2*m-1;
if ((n7 >= 0) && (n8 < 0)) n8 = rank+1;
} else if (by == DM_BOUNDARY_PERIODIC) { /* Handle Top and Bottom Sides */
if (n1 < 0) n1 = rank + m * (n-1);
if (n7 < 0) n7 = rank - m * (n-1);
if ((n3 >= 0) && (n0 < 0)) n0 = size - m + rank - 1;
if ((n3 >= 0) && (n6 < 0)) n6 = (rank%m)-1;
if ((n5 >= 0) && (n2 < 0)) n2 = size - m + rank + 1;
if ((n5 >= 0) && (n8 < 0)) n8 = (rank%m)+1;
} else if (bx == DM_BOUNDARY_PERIODIC) { /* Handle Left and Right Sides */
if (n3 < 0) n3 = rank + (m-1);
if (n5 < 0) n5 = rank - (m-1);
if ((n1 >= 0) && (n0 < 0)) n0 = rank-1;
if ((n1 >= 0) && (n2 < 0)) n2 = rank-2*m+1;
if ((n7 >= 0) && (n6 < 0)) n6 = rank+2*m-1;
if ((n7 >= 0) && (n8 < 0)) n8 = rank+1;
}
ierr = PetscMalloc1(9,&dd->neighbors);CHKERRQ(ierr);
dd->neighbors[0] = n0;
dd->neighbors[1] = n1;
dd->neighbors[2] = n2;
dd->neighbors[3] = n3;
dd->neighbors[4] = rank;
dd->neighbors[5] = n5;
dd->neighbors[6] = n6;
dd->neighbors[7] = n7;
dd->neighbors[8] = n8;
if (stencil_type == DMDA_STENCIL_STAR) {
/* save corner processor numbers */
sn0 = n0; sn2 = n2; sn6 = n6; sn8 = n8;
n0 = n2 = n6 = n8 = -1;
}
ierr = PetscMalloc1((Xe-Xs)*(Ye-Ys),&idx);CHKERRQ(ierr);
nn = 0;
xbase = bases[rank];
for (i=1; i<=s_y; i++) {
if (n0 >= 0) { /* left below */
x_t = lx[n0 % m];
y_t = ly[(n0/m)];
s_t = bases[n0] + x_t*y_t - (s_y-i)*x_t - s_x;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
}
if (n1 >= 0) { /* directly below */
x_t = x;
y_t = ly[(n1/m)];
s_t = bases[n1] + x_t*y_t - (s_y+1-i)*x_t;
for (j=0; j<x_t; j++) idx[nn++] = s_t++;
} else if (by == DM_BOUNDARY_MIRROR) {
for (j=0; j<x; j++) idx[nn++] = bases[rank] + x*(s_y - i + 1) + j;
}
if (n2 >= 0) { /* right below */
x_t = lx[n2 % m];
y_t = ly[(n2/m)];
s_t = bases[n2] + x_t*y_t - (s_y+1-i)*x_t;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
}
}
for (i=0; i<y; i++) {
if (n3 >= 0) { /* directly left */
x_t = lx[n3 % m];
/* y_t = y; */
s_t = bases[n3] + (i+1)*x_t - s_x;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
} else if (bx == DM_BOUNDARY_MIRROR) {
for (j=0; j<s_x; j++) idx[nn++] = bases[rank] + x*i + s_x - j;
}
for (j=0; j<x; j++) idx[nn++] = xbase++; /* interior */
if (n5 >= 0) { /* directly right */
x_t = lx[n5 % m];
/* y_t = y; */
s_t = bases[n5] + (i)*x_t;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
} else if (bx == DM_BOUNDARY_MIRROR) {
for (j=0; j<s_x; j++) idx[nn++] = bases[rank] + x*(i + 1) - 2 - j;
}
}
for (i=1; i<=s_y; i++) {
if (n6 >= 0) { /* left above */
x_t = lx[n6 % m];
/* y_t = ly[(n6/m)]; */
s_t = bases[n6] + (i)*x_t - s_x;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
}
if (n7 >= 0) { /* directly above */
x_t = x;
/* y_t = ly[(n7/m)]; */
s_t = bases[n7] + (i-1)*x_t;
for (j=0; j<x_t; j++) idx[nn++] = s_t++;
} else if (by == DM_BOUNDARY_MIRROR) {
for (j=0; j<x; j++) idx[nn++] = bases[rank] + x*(y - i - 1) + j;
}
if (n8 >= 0) { /* right above */
x_t = lx[n8 % m];
/* y_t = ly[(n8/m)]; */
s_t = bases[n8] + (i-1)*x_t;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
}
}
ierr = ISCreateBlock(comm,dof,nn,idx,PETSC_USE_POINTER,&from);CHKERRQ(ierr);
ierr = VecScatterCreate(global,from,local,to,>ol);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)da,(PetscObject)gtol);CHKERRQ(ierr);
ierr = ISDestroy(&to);CHKERRQ(ierr);
ierr = ISDestroy(&from);CHKERRQ(ierr);
if (stencil_type == DMDA_STENCIL_STAR) {
n0 = sn0; n2 = sn2; n6 = sn6; n8 = sn8;
}
if (((stencil_type == DMDA_STENCIL_STAR) ||
(bx && bx != DM_BOUNDARY_PERIODIC) ||
(by && by != DM_BOUNDARY_PERIODIC))) {
/*
Recompute the local to global mappings, this time keeping the
information about the cross corner processor numbers and any ghosted
but not periodic indices.
*/
nn = 0;
xbase = bases[rank];
for (i=1; i<=s_y; i++) {
if (n0 >= 0) { /* left below */
x_t = lx[n0 % m];
y_t = ly[(n0/m)];
s_t = bases[n0] + x_t*y_t - (s_y-i)*x_t - s_x;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
} else if (xs-Xs > 0 && ys-Ys > 0) {
for (j=0; j<s_x; j++) idx[nn++] = -1;
}
if (n1 >= 0) { /* directly below */
x_t = x;
y_t = ly[(n1/m)];
s_t = bases[n1] + x_t*y_t - (s_y+1-i)*x_t;
for (j=0; j<x_t; j++) idx[nn++] = s_t++;
} else if (ys-Ys > 0) {
if (by == DM_BOUNDARY_MIRROR) {
for (j=0; j<x; j++) idx[nn++] = bases[rank] + x*(s_y - i + 1) + j;
} else {
for (j=0; j<x; j++) idx[nn++] = -1;
}
}
if (n2 >= 0) { /* right below */
x_t = lx[n2 % m];
y_t = ly[(n2/m)];
s_t = bases[n2] + x_t*y_t - (s_y+1-i)*x_t;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
} else if (Xe-xe> 0 && ys-Ys > 0) {
for (j=0; j<s_x; j++) idx[nn++] = -1;
}
}
for (i=0; i<y; i++) {
if (n3 >= 0) { /* directly left */
x_t = lx[n3 % m];
/* y_t = y; */
s_t = bases[n3] + (i+1)*x_t - s_x;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
} else if (xs-Xs > 0) {
if (bx == DM_BOUNDARY_MIRROR) {
for (j=0; j<s_x; j++) idx[nn++] = bases[rank] + x*i + s_x - j;
} else {
for (j=0; j<s_x; j++) idx[nn++] = -1;
}
}
for (j=0; j<x; j++) idx[nn++] = xbase++; /* interior */
if (n5 >= 0) { /* directly right */
x_t = lx[n5 % m];
/* y_t = y; */
s_t = bases[n5] + (i)*x_t;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
} else if (Xe-xe > 0) {
if (bx == DM_BOUNDARY_MIRROR) {
for (j=0; j<s_x; j++) idx[nn++] = bases[rank] + x*(i + 1) - 2 - j;
} else {
for (j=0; j<s_x; j++) idx[nn++] = -1;
}
}
}
for (i=1; i<=s_y; i++) {
if (n6 >= 0) { /* left above */
x_t = lx[n6 % m];
/* y_t = ly[(n6/m)]; */
s_t = bases[n6] + (i)*x_t - s_x;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
} else if (xs-Xs > 0 && Ye-ye > 0) {
for (j=0; j<s_x; j++) idx[nn++] = -1;
}
if (n7 >= 0) { /* directly above */
x_t = x;
/* y_t = ly[(n7/m)]; */
s_t = bases[n7] + (i-1)*x_t;
for (j=0; j<x_t; j++) idx[nn++] = s_t++;
} else if (Ye-ye > 0) {
if (by == DM_BOUNDARY_MIRROR) {
for (j=0; j<x; j++) idx[nn++] = bases[rank] + x*(y - i - 1) + j;
} else {
for (j=0; j<x; j++) idx[nn++] = -1;
}
}
if (n8 >= 0) { /* right above */
x_t = lx[n8 % m];
/* y_t = ly[(n8/m)]; */
s_t = bases[n8] + (i-1)*x_t;
for (j=0; j<s_x; j++) idx[nn++] = s_t++;
} else if (Xe-xe > 0 && Ye-ye > 0) {
for (j=0; j<s_x; j++) idx[nn++] = -1;
}
}
}
/*
Set the local to global ordering in the global vector, this allows use
of VecSetValuesLocal().
*/
ierr = ISLocalToGlobalMappingCreate(comm,dof,nn,idx,PETSC_OWN_POINTER,&da->ltogmap);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)da,(PetscObject)da->ltogmap);CHKERRQ(ierr);
ierr = PetscFree2(bases,ldims);CHKERRQ(ierr);
dd->m = m; dd->n = n;
/* note petsc expects xs/xe/Xs/Xe to be multiplied by #dofs in many places */
dd->xs = xs*dof; dd->xe = xe*dof; dd->ys = ys; dd->ye = ye; dd->zs = 0; dd->ze = 1;
dd->Xs = Xs*dof; dd->Xe = Xe*dof; dd->Ys = Ys; dd->Ye = Ye; dd->Zs = 0; dd->Ze = 1;
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
dd->gtol = gtol;
dd->base = base;
da->ops->view = DMView_DA_2d;
dd->ltol = NULL;
dd->ao = NULL;
PetscFunctionReturn(0);
}
/*@C
SlicedGetMatrix - Creates a matrix with the correct parallel layout required for
computing the Jacobian on a function defined using the informatin in Sliced.
Collective on Sliced
Input Parameter:
+ slice - the slice object
- mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, MATSEQSBAIJ, MATMPISBAIJ,
or any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).
Output Parameters:
. J - matrix with the correct nonzero preallocation
(obviously without the correct Jacobian values)
Level: advanced
Notes: This properly preallocates the number of nonzeros in the sparse matrix so you
do not need to do it yourself.
.seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), DASetBlockFills()
@*/
PetscErrorCode PETSCDM_DLLEXPORT SlicedGetMatrix(Sliced slice, const MatType mtype,Mat *J)
{
PetscErrorCode ierr;
PetscInt *globals,*sd_nnz,*so_nnz,rstart,bs,i;
ISLocalToGlobalMapping lmap,blmap;
void (*aij)(void) = PETSC_NULL;
PetscFunctionBegin;
bs = slice->bs;
ierr = MatCreate(((PetscObject)slice)->comm,J);CHKERRQ(ierr);
ierr = MatSetSizes(*J,slice->n*bs,slice->n*bs,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(*J,mtype);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(*J,bs,slice->d_nz,slice->d_nnz);CHKERRQ(ierr);
ierr = MatMPIBAIJSetPreallocation(*J,bs,slice->d_nz,slice->d_nnz,slice->o_nz,slice->o_nnz);CHKERRQ(ierr);
/* In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
* good before going on with it. */
ierr = PetscObjectQueryFunction((PetscObject)*J,"MatMPIAIJSetPreallocation_C",&aij);CHKERRQ(ierr);
if (!aij) {
ierr = PetscObjectQueryFunction((PetscObject)*J,"MatSeqAIJSetPreallocation_C",&aij);CHKERRQ(ierr);
}
if (aij) {
if (bs == 1) {
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz,slice->d_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz,slice->d_nnz,slice->o_nz,slice->o_nnz);CHKERRQ(ierr);
} else if (!slice->d_nnz) {
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz*bs,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz*bs,PETSC_NULL,slice->o_nz*bs,PETSC_NULL);CHKERRQ(ierr);
} else {
/* The user has provided preallocation per block-row, convert it to per scalar-row respecting SlicedSetBlockFills() if applicable */
ierr = PetscMalloc2(slice->n*bs,PetscInt,&sd_nnz,(!!slice->o_nnz)*slice->n*bs,PetscInt,&so_nnz);CHKERRQ(ierr);
for (i=0; i<slice->n*bs; i++) {
sd_nnz[i] = (slice->d_nnz[i/bs]-1) * (slice->ofill ? slice->ofill->i[i%bs+1]-slice->ofill->i[i%bs] : bs)
+ (slice->dfill ? slice->dfill->i[i%bs+1]-slice->dfill->i[i%bs] : bs);
if (so_nnz) {
so_nnz[i] = slice->o_nnz[i/bs] * (slice->ofill ? slice->ofill->i[i%bs+1]-slice->ofill->i[i%bs] : bs);
}
}
ierr = MatSeqAIJSetPreallocation(*J,slice->d_nz*bs,sd_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*J,slice->d_nz*bs,sd_nnz,slice->o_nz*bs,so_nnz);CHKERRQ(ierr);
ierr = PetscFree2(sd_nnz,so_nnz);CHKERRQ(ierr);
}
}
ierr = MatSetBlockSize(*J,bs);CHKERRQ(ierr);
/* Set up the local to global map. For the scalar map, we have to translate to entry-wise indexing instead of block-wise. */
ierr = PetscMalloc((slice->n+slice->Nghosts)*bs*sizeof(PetscInt),&globals);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(*J,&rstart,PETSC_NULL);CHKERRQ(ierr);
for (i=0; i<slice->n*bs; i++) {
globals[i] = rstart + i;
}
for (i=0; i<slice->Nghosts*bs; i++) {
globals[slice->n*bs+i] = slice->ghosts[i/bs]*bs + i%bs;
}
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,(slice->n+slice->Nghosts)*bs,globals,&lmap);CHKERRQ(ierr);
ierr = PetscFree(globals);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingBlock(lmap,bs,&blmap);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMapping(*J,lmap);CHKERRQ(ierr);
ierr = MatSetLocalToGlobalMappingBlock(*J,blmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(lmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(blmap);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_INTERN PetscErrorCode DMSetUp_Stag_1d(DM dm)
{
PetscErrorCode ierr;
DM_Stag * const stag = (DM_Stag*)dm->data;
PetscMPIInt size,rank;
MPI_Comm comm;
PetscInt j;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
/* Check Global size */
if (stag->N[0] < 1) SETERRQ1(comm,PETSC_ERR_ARG_OUTOFRANGE,"Global grid size of %D < 1 specified",stag->N[0]);
/* Local sizes */
if (stag->N[0] < size) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"More ranks (%d) than elements (%D) specified",size,stag->N[0]);
if (!stag->l[0]) {
/* Divide equally, giving an extra elements to higher ranks */
ierr = PetscMalloc1(stag->nRanks[0],&stag->l[0]);CHKERRQ(ierr);
for (j=0; j<stag->nRanks[0]; ++j) stag->l[0][j] = stag->N[0]/stag->nRanks[0] + (stag->N[0] % stag->nRanks[0] > j ? 1 : 0);
}
{
PetscInt Nchk = 0;
for (j=0; j<size; ++j) Nchk += stag->l[0][j];
if (Nchk != stag->N[0]) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"Sum of specified local sizes (%D) is not equal to global size (%D)",Nchk,stag->N[0]);
}
stag->n[0] = stag->l[0][rank];
/* Rank (trivial in 1d) */
stag->rank[0] = rank;
stag->firstRank[0] = (PetscBool)(rank == 0);
stag->lastRank[0] = (PetscBool)(rank == size-1);
/* Local (unghosted) numbers of entries */
stag->entriesPerElement = stag->dof[0] + stag->dof[1];
switch (stag->boundaryType[0]) {
case DM_BOUNDARY_NONE:
case DM_BOUNDARY_GHOSTED: stag->entries = stag->n[0] * stag->entriesPerElement + (stag->lastRank[0] ? stag->dof[0] : 0); break;
case DM_BOUNDARY_PERIODIC: stag->entries = stag->n[0] * stag->entriesPerElement; break;
default: SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Unsupported x boundary type %s",DMBoundaryTypes[stag->boundaryType[0]]);
}
/* Starting element */
stag->start[0] = 0;
for(j=0; j<stag->rank[0]; ++j) stag->start[0] += stag->l[0][j];
/* Local/ghosted size and starting element */
switch (stag->boundaryType[0]) {
case DM_BOUNDARY_NONE :
switch (stag->stencilType) {
case DMSTAG_STENCIL_NONE : /* Only dummy cells on the right */
stag->startGhost[0] = stag->start[0];
stag->nGhost[0] = stag->n[0] + (stag->lastRank[0] ? 1 : 0);
break;
case DMSTAG_STENCIL_STAR :
case DMSTAG_STENCIL_BOX :
stag->startGhost[0] = stag->firstRank[0] ? stag->start[0]: stag->start[0] - stag->stencilWidth;
stag->nGhost[0] = stag->n[0];
stag->nGhost[0] += stag->firstRank[0] ? 0 : stag->stencilWidth;
stag->nGhost[0] += stag->lastRank[0] ? 1 : stag->stencilWidth;
break;
default :
SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Unrecognized ghost stencil type %d",stag->stencilType);
}
break;
case DM_BOUNDARY_GHOSTED:
case DM_BOUNDARY_PERIODIC:
switch (stag->stencilType) {
case DMSTAG_STENCIL_NONE :
stag->startGhost[0] = stag->start[0];
stag->nGhost[0] = stag->n[0];
break;
case DMSTAG_STENCIL_STAR :
case DMSTAG_STENCIL_BOX :
stag->startGhost[0] = stag->start[0] - stag->stencilWidth; /* Note that this value may be negative */
stag->nGhost[0] = stag->n[0] + 2*stag->stencilWidth;
break;
default :
SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Unrecognized ghost stencil type %d",stag->stencilType);
}
break;
default :
SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Unsupported x boundary type %s",DMBoundaryTypes[stag->boundaryType[0]]);
}
/* Total size of ghosted/local represention */
stag->entriesGhost = stag->nGhost[0]*stag->entriesPerElement;
/* Define neighbors */
ierr = PetscMalloc1(3,&stag->neighbors);CHKERRQ(ierr);
if (stag->firstRank[0]) {
switch (stag->boundaryType[0]) {
case DM_BOUNDARY_GHOSTED:
case DM_BOUNDARY_NONE: stag->neighbors[0] = -1; break;
case DM_BOUNDARY_PERIODIC: stag->neighbors[0] = stag->nRanks[0]-1; break;
default : SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Unsupported x boundary type %s",DMBoundaryTypes[stag->boundaryType[0]]);
}
} else {
stag->neighbors[0] = stag->rank[0]-1;
}
stag->neighbors[1] = stag->rank[0];
if (stag->lastRank[0]) {
switch (stag->boundaryType[0]) {
case DM_BOUNDARY_GHOSTED:
case DM_BOUNDARY_NONE: stag->neighbors[2] = -1; break;
case DM_BOUNDARY_PERIODIC: stag->neighbors[2] = 0; break;
default : SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Unsupported x boundary type %s",DMBoundaryTypes[stag->boundaryType[0]]);
}
} else {
stag->neighbors[2] = stag->rank[0]+1;
}
if (stag->n[0] < stag->stencilWidth) {
SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_SUP,"DMStag 1d setup does not support local sizes (%d) smaller than the elementwise stencil width (%d)",stag->n[0],stag->stencilWidth);
}
/* Create global->local VecScatter and ISLocalToGlobalMapping */
{
PetscInt *idxLocal,*idxGlobal,*idxGlobalAll;
PetscInt i,iLocal,d,entriesToTransferTotal,ghostOffsetStart,ghostOffsetEnd,nNonDummyGhost;
IS isLocal,isGlobal;
/* The offset on the right (may not be equal to the stencil width, as we
always have at least one ghost element, to account for the boundary
point, and may with ghosted boundaries), and the number of non-dummy ghost elements */
ghostOffsetStart = stag->start[0] - stag->startGhost[0];
ghostOffsetEnd = stag->startGhost[0]+stag->nGhost[0] - (stag->start[0]+stag->n[0]);
nNonDummyGhost = stag->nGhost[0] - (stag->lastRank[0] ? ghostOffsetEnd : 0) - (stag->firstRank[0] ? ghostOffsetStart : 0);
/* Compute the number of non-dummy entries in the local representation
This is equal to the number of non-dummy elements in the local (ghosted) representation,
plus some extra entries on the right boundary on the last rank*/
switch (stag->boundaryType[0]) {
case DM_BOUNDARY_GHOSTED:
case DM_BOUNDARY_NONE:
entriesToTransferTotal = nNonDummyGhost * stag->entriesPerElement + (stag->lastRank[0] ? stag->dof[0] : 0);
break;
case DM_BOUNDARY_PERIODIC:
entriesToTransferTotal = stag->entriesGhost; /* No dummy points */
break;
default :
SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Unsupported x boundary type %s",DMBoundaryTypes[stag->boundaryType[0]]);
}
ierr = PetscMalloc1(entriesToTransferTotal,&idxLocal);CHKERRQ(ierr);
ierr = PetscMalloc1(entriesToTransferTotal,&idxGlobal);CHKERRQ(ierr);
ierr = PetscMalloc1(stag->entriesGhost,&idxGlobalAll);CHKERRQ(ierr);
if (stag->boundaryType[0] == DM_BOUNDARY_NONE) {
PetscInt count = 0,countAll = 0;
/* Left ghost points and native points */
for (i=stag->startGhost[0], iLocal=0; iLocal<nNonDummyGhost; ++i,++iLocal) {
for (d=0; d<stag->entriesPerElement; ++d,++count,++countAll) {
idxLocal [count] = iLocal * stag->entriesPerElement + d;
idxGlobal[count] = i * stag->entriesPerElement + d;
idxGlobalAll[countAll] = i * stag->entriesPerElement + d;
}
}
/* Ghost points on the right
Special case for last (partial dummy) element on the last rank */
if (stag->lastRank[0] ) {
i = stag->N[0];
iLocal = (stag->nGhost[0]-ghostOffsetEnd);
/* Only vertex (0-cell) dofs in global representation */
for (d=0; d<stag->dof[0]; ++d,++count,++countAll) {
idxGlobal[count] = i * stag->entriesPerElement + d;
idxLocal [count] = iLocal * stag->entriesPerElement + d;
idxGlobalAll[countAll] = i * stag->entriesPerElement + d;
}
for (d=stag->dof[0]; d<stag->entriesPerElement; ++d,++countAll) { /* Additional dummy entries */
idxGlobalAll[countAll] = -1;
}
}
} else if (stag->boundaryType[0] == DM_BOUNDARY_PERIODIC) {
PetscInt count = 0,iLocal = 0; /* No dummy points, so idxGlobal and idxGlobalAll are identical */
const PetscInt iMin = stag->firstRank[0] ? stag->start[0] : stag->startGhost[0];
const PetscInt iMax = stag->lastRank[0] ? stag->startGhost[0] + stag->nGhost[0] - stag->stencilWidth : stag->startGhost[0] + stag->nGhost[0];
/* Ghost points on the left */
if (stag->firstRank[0]) {
for (i=stag->N[0]-stag->stencilWidth; iLocal<stag->stencilWidth; ++i,++iLocal) {
for (d=0; d<stag->entriesPerElement; ++d,++count) {
idxGlobal[count] = i * stag->entriesPerElement + d;
idxLocal [count] = iLocal * stag->entriesPerElement + d;
idxGlobalAll[count] = idxGlobal[count];
}
}
}
/* Native points */
for (i=iMin; i<iMax; ++i,++iLocal) {
for (d=0; d<stag->entriesPerElement; ++d,++count) {
idxGlobal[count] = i * stag->entriesPerElement + d;
idxLocal [count] = iLocal * stag->entriesPerElement + d;
idxGlobalAll[count] = idxGlobal[count];
}
}
/* Ghost points on the right */
if (stag->lastRank[0]) {
for (i=0; iLocal<stag->nGhost[0]; ++i,++iLocal) {
for (d=0; d<stag->entriesPerElement; ++d,++count) {
idxGlobal[count] = i * stag->entriesPerElement + d;
idxLocal [count] = iLocal * stag->entriesPerElement + d;
idxGlobalAll[count] = idxGlobal[count];
}
}
}
} else if (stag->boundaryType[0] == DM_BOUNDARY_GHOSTED) {
PetscInt count = 0,countAll = 0;
/* Dummy elements on the left, on the first rank */
if (stag->firstRank[0]) {
for(iLocal=0; iLocal<ghostOffsetStart; ++iLocal) {
/* Complete elements full of dummy entries */
for (d=0; d<stag->entriesPerElement; ++d,++countAll) {
idxGlobalAll[countAll] = -1;
}
}
i = 0; /* nonDummy entries start with global entry 0 */
} else {
/* nonDummy entries start as usual */
i = stag->startGhost[0];
iLocal = 0;
}
/* non-Dummy entries */
{
PetscInt iLocalNonDummyMax = stag->firstRank[0] ? nNonDummyGhost + ghostOffsetStart : nNonDummyGhost;
for (; iLocal<iLocalNonDummyMax; ++i,++iLocal) {
for (d=0; d<stag->entriesPerElement; ++d,++count,++countAll) {
idxLocal [count] = iLocal * stag->entriesPerElement + d;
idxGlobal[count] = i * stag->entriesPerElement + d;
idxGlobalAll[countAll] = i * stag->entriesPerElement + d;
}
}
}
/* (partial) dummy elements on the right, on the last rank */
if (stag->lastRank[0]) {
/* First one is partial dummy */
i = stag->N[0];
iLocal = (stag->nGhost[0]-ghostOffsetEnd);
for (d=0; d<stag->dof[0]; ++d,++count,++countAll) { /* Only vertex (0-cell) dofs in global representation */
idxLocal [count] = iLocal * stag->entriesPerElement + d;
idxGlobal[count] = i * stag->entriesPerElement + d;
idxGlobalAll[countAll] = i * stag->entriesPerElement + d;
}
for (d=stag->dof[0]; d<stag->entriesPerElement; ++d,++countAll) { /* Additional dummy entries */
idxGlobalAll[countAll] = -1;
}
for (iLocal = stag->nGhost[0] - ghostOffsetEnd + 1; iLocal < stag->nGhost[0]; ++iLocal) {
/* Additional dummy elements */
for (d=0; d<stag->entriesPerElement; ++d,++countAll) {
idxGlobalAll[countAll] = -1;
}
}
}
} else SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Unsupported x boundary type %s",DMBoundaryTypes[stag->boundaryType[0]]);
/* Create Local IS (transferring pointer ownership) */
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm),entriesToTransferTotal,idxLocal,PETSC_OWN_POINTER,&isLocal);CHKERRQ(ierr);
/* Create Global IS (transferring pointer ownership) */
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)dm),entriesToTransferTotal,idxGlobal,PETSC_OWN_POINTER,&isGlobal);CHKERRQ(ierr);
/* Create stag->gtol, which doesn't include dummy entries */
{
Vec local,global;
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)dm),1,stag->entries,PETSC_DECIDE,NULL,&global);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF,stag->entriesPerElement,stag->entriesGhost,NULL,&local);CHKERRQ(ierr);
ierr = VecScatterCreateWithData(global,isGlobal,local,isLocal,&stag->gtol);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
}
/* Destroy ISs */
ierr = ISDestroy(&isLocal);CHKERRQ(ierr);
ierr = ISDestroy(&isGlobal);CHKERRQ(ierr);
/* Create local-to-global map (transferring pointer ownership) */
ierr = ISLocalToGlobalMappingCreate(comm,1,stag->entriesGhost,idxGlobalAll,PETSC_OWN_POINTER,&dm->ltogmap);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)dm,(PetscObject)dm->ltogmap);CHKERRQ(ierr);
}
/* Precompute location offsets */
ierr = DMStagComputeLocationOffsets_1d(dm);CHKERRQ(ierr);
/* View from Options */
ierr = DMViewFromOptions(dm,NULL,"-dm_view");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt rank;
PetscInt i,N,ng,*gindices,rstart,rend,M;
PetscScalar one = 1.0;
Vec x;
PetscInitialize(&argc,&argv,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/*
Create a parallel vector.
- In this case, we specify the size of each processor's local
portion, and PETSc computes the global size. Alternatively,
PETSc could determine the vector's distribution if we specify
just the global size.
*/
ierr = VecCreate(PETSC_COMM_WORLD,&x);CHKERRQ(ierr);
ierr = VecSetSizes(x,rank+1,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(x);CHKERRQ(ierr);
ierr = VecGetSize(x,&N);CHKERRQ(ierr);
ierr = VecSet(x,one);CHKERRQ(ierr);
/*
Set the local to global ordering for the vector. Each processor
generates a list of the global indices for each local index. Note that
the local indices are just whatever is convenient for a particular application.
In this case we treat the vector as lying on a one dimensional grid and
have one ghost point on each end of the blocks owned by each processor.
*/
ierr = VecGetSize(x,&M);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x,&rstart,&rend);CHKERRQ(ierr);
ng = rend - rstart + 2;
ierr = PetscMalloc(ng*sizeof(PetscInt),&gindices);CHKERRQ(ierr);
gindices[0] = rstart - 1;
for (i=0; i<ng-1; i++) {
gindices[i+1] = gindices[i] + 1;
}
/* map the first and last point as periodic */
if (gindices[0] == -1) gindices[0] = M - 1;
if (gindices[ng-1] == M) gindices[ng-1] = 0;
{
ISLocalToGlobalMapping ltog;
ierr = ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,ng,gindices,PETSC_COPY_VALUES,<og);CHKERRQ(ierr);
ierr = VecSetLocalToGlobalMapping(x,ltog);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(<og);CHKERRQ(ierr);
}
ierr = PetscFree(gindices);CHKERRQ(ierr);
/*
Set the vector elements.
- In this case set the values using the local ordering
- Each processor can contribute any vector entries,
regardless of which processor "owns" them; any nonlocal
contributions will be transferred to the appropriate processor
during the assembly process.
- In this example, the flag ADD_VALUES indicates that all
contributions will be added together.
*/
for (i=0; i<ng; i++) {
ierr = VecSetValuesLocal(x,1,&i,&one,ADD_VALUES);CHKERRQ(ierr);
}
/*
Assemble vector, using the 2-step process:
VecAssemblyBegin(), VecAssemblyEnd()
Computations can be done while messages are in transition
by placing code between these two statements.
*/
ierr = VecAssemblyBegin(x);CHKERRQ(ierr);
ierr = VecAssemblyEnd(x);CHKERRQ(ierr);
/*
View the vector; then destroy it.
*/
ierr = VecView(x,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}