PETSC_EXTERN void PETSC_STDCALL islocaltoglobalmappinggetindices_(ISLocalToGlobalMapping *x,PetscInt *fa,size_t *ia,PetscErrorCode *ierr)
{
const PetscInt *lx;
*ierr = ISLocalToGlobalMappingGetIndices(*x,&lx); if (*ierr) return;
*ia = PetscIntAddressToFortran(fa,(PetscInt*)lx);
}
/*@C
DMCompositeGetISLocalToGlobalMappings - gets an ISLocalToGlobalMapping for each DM in the DMComposite, maps to the composite global space
Collective on DM
Input Parameter:
. dm - the packer object
Output Parameters:
. ltogs - the individual mappings for each packed vector. Note that this includes
all the ghost points that individual ghosted DMDA's may have.
Level: advanced
Notes:
Each entry of ltogs should be destroyed with ISLocalToGlobalMappingDestroy(), the ltogs array should be freed with PetscFree().
.seealso DMDestroy(), DMCompositeAddDM(), DMCreateGlobalVector(),
DMCompositeGather(), DMCompositeCreate(), DMCompositeGetAccess(), DMCompositeScatter(),
DMCompositeGetLocalVectors(), DMCompositeRestoreLocalVectors(),DMCompositeGetEntries()
@*/
PetscErrorCode DMCompositeGetISLocalToGlobalMappings(DM dm,ISLocalToGlobalMapping **ltogs)
{
PetscErrorCode ierr;
PetscInt i,*idx,n,cnt;
struct DMCompositeLink *next;
PetscMPIInt rank;
DM_Composite *com = (DM_Composite*)dm->data;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm,DM_CLASSID,1);
ierr = DMSetUp(dm);CHKERRQ(ierr);
ierr = PetscMalloc((com->nDM)*sizeof(ISLocalToGlobalMapping),ltogs);CHKERRQ(ierr);
next = com->next;
ierr = MPI_Comm_rank(((PetscObject)dm)->comm,&rank);CHKERRQ(ierr);
/* loop over packed objects, handling one at at time */
cnt = 0;
while (next) {
ISLocalToGlobalMapping ltog;
PetscMPIInt size;
const PetscInt *suboff,*indices;
Vec global;
/* Get sub-DM global indices for each local dof */
ierr = DMGetLocalToGlobalMapping(next->dm,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltog,&n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog,&indices);CHKERRQ(ierr);
ierr = PetscMalloc(n*sizeof(PetscInt),&idx);CHKERRQ(ierr);
/* Get the offsets for the sub-DM global vector */
ierr = DMGetGlobalVector(next->dm,&global);CHKERRQ(ierr);
ierr = VecGetOwnershipRanges(global,&suboff);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)global)->comm,&size);CHKERRQ(ierr);
/* Shift the sub-DM definition of the global space to the composite global space */
for (i=0; i<n; i++) {
PetscInt subi = indices[i],lo = 0,hi = size,t;
/* Binary search to find which rank owns subi */
while (hi-lo > 1) {
t = lo + (hi-lo)/2;
if (suboff[t] > subi) hi = t;
else lo = t;
}
idx[i] = subi - suboff[lo] + next->grstarts[lo];
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltog,&indices);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm,n,idx,PETSC_OWN_POINTER,&(*ltogs)[cnt]);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(next->dm,&global);CHKERRQ(ierr);
next = next->next;
cnt++;
}
PetscFunctionReturn(0);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscSF sf;
PetscInt bs,i,nroots,*rootdata,nleaves,*leafdata,nlocalcols;
const PetscInt *gidxs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) {
SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
}
ierr = MatGetLocalSize(B,&nroots,NULL);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&nleaves,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscCalloc2(nroots,&rootdata,nleaves,&leafdata);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)B),&sf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,B->rmap,nleaves,NULL,PETSC_COPY_VALUES,gidxs);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
if (!d_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += d_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += o_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves;i++) {
leafdata[i] = PetscMin(leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree2(rootdata,leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int FormJacobian_Grid(GridCtx *grid,Mat *J)
{
Mat jac = *J;
PetscErrorCode ierr;
PetscInt i,j,row,mx,my,xs,ys,xm,ym,Xs,Ys,Xm,Ym,col[5];
PetscInt grow;
const PetscInt *ltog;
PetscScalar two = 2.0,one = 1.0,v[5],hx,hy,hxdhy,hydhx,value;
ISLocalToGlobalMapping ltogm;
mx = grid->mx; my = grid->my;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
hxdhy = hx/hy; hydhx = hy/hx;
/* Get ghost points */
ierr = DMDAGetCorners(grid->da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(grid->da,&Xs,&Ys,0,&Xm,&Ym,0);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(grid->da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
/* Evaluate Jacobian of function */
for (j=ys; j<ys+ym; j++) {
row = (j - Ys)*Xm + xs - Xs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
if (i > 0 && i < mx-1 && j > 0 && j < my-1) {
v[0] = -hxdhy; col[0] = ltog[row - Xm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + Xm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
} else if ((i > 0 && i < mx-1) || (j > 0 && j < my-1)) {
value = .5*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
} else {
value = .25*two*(hydhx + hxdhy);
ierr = MatSetValues(jac,1,&grow,1,&grow,&value,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
static PetscErrorCode MatISComputeSF_Private(Mat B)
{
Mat_IS *matis = (Mat_IS*)(B->data);
const PetscInt *gidxs;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MatGetSize(matis->A,&matis->sf_nleaves,NULL);CHKERRQ(ierr);
ierr = MatGetLocalSize(B,&matis->sf_nroots,NULL);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)B),&matis->sf);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(B->rmap->mapping,&gidxs);CHKERRQ(ierr);
/* PETSC_OWN_POINTER refers to ilocal which is NULL */
ierr = PetscSFSetGraphLayout(matis->sf,B->rmap,matis->sf_nleaves,NULL,PETSC_OWN_POINTER,gidxs);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(B->rmap->mapping,&gidxs);CHKERRQ(ierr);
ierr = PetscMalloc2(matis->sf_nroots,&matis->sf_rootdata,matis->sf_nleaves,&matis->sf_leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
Vec CreateGhostedVector(Grid& g) {
auto dist = boost::any_cast<const Decomposition&>(g.userData());
int n;
ISLocalToGlobalMappingGetSize(dist.locToGlobMap, &n);
const PetscInt *idx;
ISLocalToGlobalMappingGetIndices(dist.locToGlobMap, &idx);
Vec v;
PetscInt nLoc = dist.cellEnd - dist.cellStart;
std::cout << n-nLoc << std::endl;
VecCreateGhost(PETSC_COMM_WORLD, g.cells().size(), PETSC_DECIDE,
n - nLoc, idx + nLoc, &v);
ISLocalToGlobalMappingRestoreIndices(dist.locToGlobMap, &idx);
return v;
}
/*
ComputeJacobian - Evaluates Jacobian matrix.
Input Parameters:
. x - input vector
. user - user-defined application context
Output Parameters:
. jac - Jacobian matrix
. flag - flag indicating matrix structure
Notes:
Due to grid point reordering with DMDAs, we must always work
with the local grid points, and then transform them to the new
global numbering with the "ltog" mapping
We cannot work directly with the global numbers for the original
uniprocessor grid!
*/
PetscErrorCode ComputeJacobian(AppCtx *user,Vec X,Mat jac)
{
PetscErrorCode ierr;
Vec localX = user->localX; /* local vector */
const PetscInt *ltog; /* local-to-global mapping */
PetscInt i,j,row,mx,my,col[5];
PetscInt xs,ys,xm,ym,gxs,gys,gxm,gym,grow;
PetscScalar two = 2.0,one = 1.0,lambda,v[5],hx,hy,hxdhy,hydhx,sc,*x;
ISLocalToGlobalMapping ltogm;
mx = user->mx; my = user->my; lambda = user->param;
hx = one/(PetscReal)(mx-1); hy = one/(PetscReal)(my-1);
sc = hx*hy; hxdhy = hx/hy; hydhx = hy/hx;
/*
Scatter ghost points to local vector, using the 2-step process
DMGlobalToLocalBegin(), DMGlobalToLocalEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = DMGlobalToLocalBegin(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(user->da,X,INSERT_VALUES,localX);CHKERRQ(ierr);
/*
Get pointer to vector data
*/
ierr = VecGetArray(localX,&x);CHKERRQ(ierr);
/*
Get local grid boundaries
*/
ierr = DMDAGetCorners(user->da,&xs,&ys,NULL,&xm,&ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(user->da,&gxs,&gys,NULL,&gxm,&gym,NULL);CHKERRQ(ierr);
/*
Get the global node numbers for all local nodes, including ghost points
*/
ierr = DMGetLocalToGlobalMapping(user->da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
/*
Compute entries for the locally owned part of the Jacobian.
- Currently, all PETSc parallel matrix formats are partitioned by
contiguous chunks of rows across the processors. The "grow"
parameter computed below specifies the global row number
corresponding to each local grid point.
- Each processor needs to insert only elements that it owns
locally (but any non-local elements will be sent to the
appropriate processor during matrix assembly).
- Always specify global row and columns of matrix entries.
- Here, we set all entries for a particular row at once.
*/
for (j=ys; j<ys+ym; j++) {
row = (j - gys)*gxm + xs - gxs - 1;
for (i=xs; i<xs+xm; i++) {
row++;
grow = ltog[row];
/* boundary points */
if (i == 0 || j == 0 || i == mx-1 || j == my-1) {
ierr = MatSetValues(jac,1,&grow,1,&grow,&one,INSERT_VALUES);CHKERRQ(ierr);
continue;
}
/* interior grid points */
v[0] = -hxdhy; col[0] = ltog[row - gxm];
v[1] = -hydhx; col[1] = ltog[row - 1];
v[2] = two*(hydhx + hxdhy) - sc*lambda*PetscExpScalar(x[row]); col[2] = grow;
v[3] = -hydhx; col[3] = ltog[row + 1];
v[4] = -hxdhy; col[4] = ltog[row + gxm];
ierr = MatSetValues(jac,1,&grow,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
/*
Assemble matrix, using the 2-step process:
MatAssemblyBegin(), MatAssemblyEnd().
By placing code between these two statements, computations can be
done while messages are in transition.
*/
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = VecRestoreArray(localX,&x);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
return 0;
}
inline PetscVector::PetscVector (Vec v)
: _array_is_present(false),
_local_form(NULL),
_values(NULL),
_global_to_local_map(),
_destroy_vec_on_exit(false) {
this->_vec = v;
this->_is_closed = true;
this->_is_initialized = true;
/* We need to ask PETSc about the (local to global) ghost value
mapping and create the inverse mapping out of it. */
int ierr=0;
int petsc_local_size=0;
ierr = VecGetLocalSize(_vec, &petsc_local_size);
CHKERRABORT(MPI_COMM_WORLD,ierr);
// // Get the vector type from PETSc.
// const VecType type;
// ierr = VecGetType(_vec, &type);
// CHKERRABORT(MPI_COMM_WORLD,ierr);
// Get the vector type from PETSc.
// As of Petsc 3.0.0, the VecType #define lost its const-ness, so we
// need to have it in the code
#if PETSC_VERSION_LESS_THAN(3,0,0) || !PETSC_VERSION_LESS_THAN(3,4,0)
// Pre-3.0 and petsc-dev (as of October 2012) use non-const versions
VecType ptype;
#else
const VecType ptype;
#endif
ierr = VecGetType(_vec, &ptype);
CHKERRABORT(MPI_COMM_WORLD,ierr);
if((strcmp(ptype,VECSHARED) == 0) || (strcmp(ptype,VECMPI) == 0)) {
#if PETSC_VERSION_RELEASE && PETSC_VERSION_LESS_THAN(3,1,1)
ISLocalToGlobalMapping mapping = _vec->mapping;
#else
ISLocalToGlobalMapping mapping;
ierr = VecGetLocalToGlobalMapping(_vec, &mapping);
CHKERRABORT(MPI_COMM_WORLD,ierr);
#endif
// ISLocalToGlobalMapping mapping;
// ierr = VecGetLocalToGlobalMapping(_vec, &mapping);
// CHKERRABORT(MPI_COMM_WORLD,ierr);
// If is a sparsely stored vector, set up our new mapping
if (mapping) {
const unsigned int local_size = static_cast<unsigned int>(petsc_local_size);
const unsigned int ghost_begin = static_cast<unsigned int>(petsc_local_size);
#if PETSC_VERSION_RELEASE && PETSC_VERSION_LESS_THAN(3,4,0)
const numeric_index_type ghost_end = static_cast<numeric_index_type>(mapping->n);
#else
PetscInt n;
ierr = ISLocalToGlobalMappingGetSize(mapping, &n);
CHKERRABORT(MPI_COMM_WORLD,ierr);
const unsigned int ghost_end = static_cast<unsigned int>(n);
#endif
#if PETSC_VERSION_RELEASE && PETSC_VERSION_LESS_THAN(3,1,1)
const PetscInt *indices = mapping->indices;
#else
const PetscInt *indices;
ierr = ISLocalToGlobalMappingGetIndices(mapping,&indices);
CHKERRABORT(MPI_COMM_WORLD,ierr);
#endif
for(unsigned int i=ghost_begin; i<ghost_end; i++)
_global_to_local_map[indices[i]] = i-local_size;
this->_type = GHOSTED;
#if !PETSC_VERSION_RELEASE || !PETSC_VERSION_LESS_THAN(3,1,1)
ierr = ISLocalToGlobalMappingRestoreIndices(mapping, &indices);
CHKERRABORT(MPI_COMM_WORLD,ierr);
#endif
// const unsigned int ghost_end = static_cast<unsigned int>(mapping->n);
// const int *indices = mapping->indices;
// ierr = ISLocalToGlobalMappingGetIndices(mapping,&indices);
// CHKERRABORT(MPI_COMM_WORLD,ierr);
// for(unsigned int i=ghost_begin; i<ghost_end; i++)
// _global_to_local_map[indices[i]] = i-local_size;
// this->_type = GHOSTED;
}
else
this->_type = PARALLEL;
}
else
this->_type = SERIAL;
this->close();
}
int main(int argc,char **argv)
{
PetscMPIInt rank;
PetscErrorCode ierr;
PetscInt M = 10,N = 8,m = PETSC_DECIDE;
PetscInt s =2,w=2,n = PETSC_DECIDE,nloc,l,i,j,kk;
PetscInt Xs,Xm,Ys,Ym,iloc,*iglobal;
const PetscInt *ltog;
PetscInt *lx = NULL,*ly = NULL;
PetscBool testorder = PETSC_FALSE,flg;
DMBoundaryType bx = DM_BOUNDARY_NONE,by= DM_BOUNDARY_NONE;
DM da;
PetscViewer viewer;
Vec local,global;
PetscScalar value;
DMDAStencilType st = DMDA_STENCIL_BOX;
AO ao;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = PetscViewerDrawOpen(PETSC_COMM_WORLD,0,"",300,0,400,400,&viewer);CHKERRQ(ierr);
/* Readoptions */
ierr = PetscOptionsGetInt(NULL,NULL,"-NX",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-NY",&N,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-s",&s,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-w",&w,NULL);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yperiodic",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_PERIODIC;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-xghosted",&flg,NULL);CHKERRQ(ierr); if (flg) bx = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-yghosted",&flg,NULL);CHKERRQ(ierr); if (flg) by = DM_BOUNDARY_GHOSTED;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-star",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_STAR;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-box",&flg,NULL);CHKERRQ(ierr); if (flg) st = DMDA_STENCIL_BOX;
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-testorder",&testorder,NULL);CHKERRQ(ierr);
/*
Test putting two nodes in x and y on each processor, exact last processor
in x and y gets the rest.
*/
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-distribute",&flg,NULL);CHKERRQ(ierr);
if (flg) {
if (m == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -m option with -distribute option");
ierr = PetscMalloc1(m,&lx);CHKERRQ(ierr);
for (i=0; i<m-1; i++) { lx[i] = 4;}
lx[m-1] = M - 4*(m-1);
if (n == PETSC_DECIDE) SETERRQ(PETSC_COMM_WORLD,1,"Must set -n option with -distribute option");
ierr = PetscMalloc1(n,&ly);CHKERRQ(ierr);
for (i=0; i<n-1; i++) { ly[i] = 2;}
ly[n-1] = N - 2*(n-1);
}
/* Create distributed array and get vectors */
ierr = DMDACreate2d(PETSC_COMM_WORLD,bx,by,st,M,N,m,n,w,s,lx,ly,&da);CHKERRQ(ierr);
ierr = PetscFree(lx);CHKERRQ(ierr);
ierr = PetscFree(ly);CHKERRQ(ierr);
ierr = DMView(da,viewer);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(da,&global);CHKERRQ(ierr);
ierr = DMCreateLocalVector(da,&local);CHKERRQ(ierr);
/* Set global vector; send ghost points to local vectors */
value = 1;
ierr = VecSet(global,value);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
/* Scale local vectors according to processor rank; pass to global vector */
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
value = rank;
ierr = VecScale(local,value);CHKERRQ(ierr);
ierr = DMLocalToGlobalBegin(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
ierr = DMLocalToGlobalEnd(da,local,INSERT_VALUES,global);CHKERRQ(ierr);
if (!testorder) { /* turn off printing when testing ordering mappings */
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nGlobal Vectors:\n");CHKERRQ(ierr);
ierr = VecView(global,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n\n");CHKERRQ(ierr);
}
/* Send ghost points to local vectors */
ierr = DMGlobalToLocalBegin(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(da,global,INSERT_VALUES,local);CHKERRQ(ierr);
flg = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,NULL,"-local_print",&flg,NULL);CHKERRQ(ierr);
if (flg) {
PetscViewer sviewer;
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"\nLocal Vector: processor %d\n",rank);CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = VecView(local,sviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&sviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Tests mappings betweeen application/PETSc orderings */
if (testorder) {
ISLocalToGlobalMapping ltogm;
ierr = DMGetLocalToGlobalMapping(da,<ogm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(ltogm,&nloc);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltogm,<og);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(da,&Xs,&Ys,NULL,&Xm,&Ym,NULL);CHKERRQ(ierr);
ierr = DMDAGetAO(da,&ao);CHKERRQ(ierr);
ierr = PetscMalloc1(nloc,&iglobal);CHKERRQ(ierr);
/* Set iglobal to be global indices for each processor's local and ghost nodes,
using the DMDA ordering of grid points */
kk = 0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
iglobal[kk++] = ltog[iloc+l];
}
}
}
/* Map this to the application ordering (which for DMDAs is just the natural ordering
that would be used for 1 processor, numbering most rapidly by x, then y) */
ierr = AOPetscToApplication(ao,nloc,iglobal);CHKERRQ(ierr);
/* Then map the application ordering back to the PETSc DMDA ordering */
ierr = AOApplicationToPetsc(ao,nloc,iglobal);CHKERRQ(ierr);
/* Verify the mappings */
kk=0;
for (j=Ys; j<Ys+Ym; j++) {
for (i=Xs; i<Xs+Xm; i++) {
iloc = w*((j-Ys)*Xm + i-Xs);
for (l=0; l<w; l++) {
if (iglobal[kk] != ltog[iloc+l]) {
ierr = PetscFPrintf(PETSC_COMM_SELF,stdout,"[%d] Problem with mapping: j=%D, i=%D, l=%D, petsc1=%D, petsc2=%D\n",rank,j,i,l,ltog[iloc+l],iglobal[kk]);CHKERRQ(ierr);
}
kk++;
}
}
}
ierr = PetscFree(iglobal);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(ltogm,<og);CHKERRQ(ierr);
}
/* Free memory */
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecDestroy(&local);CHKERRQ(ierr);
ierr = VecDestroy(&global);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PETSC_EXTERN PetscErrorCode MatISSetMPIXAIJPreallocation_Private(Mat A, Mat B, PetscBool maxreduce)
{
Mat_IS *matis = (Mat_IS*)(A->data);
PetscInt *my_dnz,*my_onz,*dnz,*onz,*mat_ranges,*row_ownership;
const PetscInt *global_indices_r,*global_indices_c;
PetscInt i,j,bs,rows,cols;
PetscInt lrows,lcols;
PetscInt local_rows,local_cols;
PetscMPIInt nsubdomains;
PetscBool isdense,issbaij;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&nsubdomains);CHKERRQ(ierr);
ierr = MatGetSize(A,&rows,&cols);CHKERRQ(ierr);
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&local_rows,&local_cols);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr);
if (A->rmap->mapping != A->cmap->mapping) {
ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr);
} else {
global_indices_c = global_indices_r;
}
if (issbaij) {
ierr = MatGetRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
/*
An SF reduce is needed to sum up properly on shared rows.
Note that generally preallocation is not exact, since it overestimates nonzeros
*/
if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
ierr = MatISComputeSF_Private(A);CHKERRQ(ierr);
}
ierr = MatGetLocalSize(A,&lrows,&lcols);CHKERRQ(ierr);
ierr = MatPreallocateInitialize(PetscObjectComm((PetscObject)A),lrows,lcols,dnz,onz);CHKERRQ(ierr);
/* All processes need to compute entire row ownership */
ierr = PetscMalloc1(rows,&row_ownership);CHKERRQ(ierr);
ierr = MatGetOwnershipRanges(A,(const PetscInt**)&mat_ranges);CHKERRQ(ierr);
for (i=0;i<nsubdomains;i++) {
for (j=mat_ranges[i];j<mat_ranges[i+1];j++) {
row_ownership[j] = i;
}
}
/*
my_dnz and my_onz contains exact contribution to preallocation from each local mat
then, they will be summed up properly. This way, preallocation is always sufficient
*/
ierr = PetscCalloc2(local_rows,&my_dnz,local_rows,&my_onz);CHKERRQ(ierr);
/* preallocation as a MATAIJ */
if (isdense) { /* special case for dense local matrices */
for (i=0;i<local_rows;i++) {
PetscInt index_row = global_indices_r[i];
for (j=i;j<local_rows;j++) {
PetscInt owner = row_ownership[index_row];
PetscInt index_col = global_indices_c[j];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1;
} else { /* offdiag block */
my_onz[i] += 1;
}
/* same as before, interchanging rows and cols */
if (i != j) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[j] += 1;
} else {
my_onz[j] += 1;
}
}
}
}
} else { /* TODO: this could be optimized using MatGetRowIJ */
for (i=0;i<local_rows;i++) {
const PetscInt *cols;
PetscInt ncols,index_row = global_indices_r[i];
ierr = MatGetRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
PetscInt owner = row_ownership[index_row];
PetscInt index_col = global_indices_c[cols[j]];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1;
} else { /* offdiag block */
my_onz[i] += 1;
}
/* same as before, interchanging rows and cols */
if (issbaij && index_col != index_row) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[cols[j]] += 1;
} else {
my_onz[cols[j]] += 1;
}
}
}
ierr = MatRestoreRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr);
if (global_indices_c != global_indices_r) {
ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr);
}
ierr = PetscFree(row_ownership);CHKERRQ(ierr);
/* Reduce my_dnz and my_onz */
if (maxreduce) {
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr);
} else {
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr);
}
ierr = PetscFree2(my_dnz,my_onz);CHKERRQ(ierr);
/* Resize preallocation if overestimated */
for (i=0;i<lrows;i++) {
dnz[i] = PetscMin(dnz[i],lcols);
onz[i] = PetscMin(onz[i],cols-lcols);
}
/* set preallocation */
ierr = MatMPIAIJSetPreallocation(B,0,dnz,0,onz);CHKERRQ(ierr);
for (i=0;i<lrows/bs;i++) {
dnz[i] = dnz[i*bs]/bs;
onz[i] = onz[i*bs]/bs;
}
ierr = MatMPIBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatPreallocateFinalize(dnz,onz);CHKERRQ(ierr);
if (issbaij) {
ierr = MatRestoreRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode MatFDColoringSetUp_MPIXAIJ(Mat mat,ISColoring iscoloring,MatFDColoring c)
{
PetscErrorCode ierr;
PetscMPIInt size,*ncolsonproc,*disp,nn;
PetscInt i,n,nrows,nrows_i,j,k,m,ncols,col,*rowhit,cstart,cend,colb;
const PetscInt *is,*A_ci,*A_cj,*B_ci,*B_cj,*row=NULL,*ltog=NULL;
PetscInt nis=iscoloring->n,nctot,*cols;
IS *isa;
ISLocalToGlobalMapping map=mat->cmap->mapping;
PetscInt ctype=c->ctype,*spidxA,*spidxB,nz,bs,bs2,spidx;
Mat A,B;
PetscScalar *A_val,*B_val,**valaddrhit;
MatEntry *Jentry;
MatEntry2 *Jentry2;
PetscBool isBAIJ;
PetscInt bcols=c->bcols;
#if defined(PETSC_USE_CTABLE)
PetscTable colmap=NULL;
#else
PetscInt *colmap=NULL; /* local col number of off-diag col */
#endif
PetscFunctionBegin;
if (ctype == IS_COLORING_GHOSTED) {
if (!map) SETERRQ(PetscObjectComm((PetscObject)mat),PETSC_ERR_ARG_INCOMP,"When using ghosted differencing matrix must have local to global mapping provided with MatSetLocalToGlobalMapping");
ierr = ISLocalToGlobalMappingGetIndices(map,<og);CHKERRQ(ierr);
}
ierr = MatGetBlockSize(mat,&bs);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)mat,MATMPIBAIJ,&isBAIJ);CHKERRQ(ierr);
if (isBAIJ) {
Mat_MPIBAIJ *baij=(Mat_MPIBAIJ*)mat->data;
Mat_SeqBAIJ *spA,*spB;
A = baij->A; spA = (Mat_SeqBAIJ*)A->data; A_val = spA->a;
B = baij->B; spB = (Mat_SeqBAIJ*)B->data; B_val = spB->a;
nz = spA->nz + spB->nz; /* total nonzero entries of mat */
if (!baij->colmap) {
ierr = MatCreateColmap_MPIBAIJ_Private(mat);CHKERRQ(ierr);
colmap = baij->colmap;
}
ierr = MatGetColumnIJ_SeqBAIJ_Color(A,0,PETSC_FALSE,PETSC_FALSE,&ncols,&A_ci,&A_cj,&spidxA,NULL);CHKERRQ(ierr);
ierr = MatGetColumnIJ_SeqBAIJ_Color(B,0,PETSC_FALSE,PETSC_FALSE,&ncols,&B_ci,&B_cj,&spidxB,NULL);CHKERRQ(ierr);
if (ctype == IS_COLORING_GLOBAL && c->htype[0] == 'd') { /* create vscale for storing dx */
PetscInt *garray;
ierr = PetscMalloc1(B->cmap->n,&garray);CHKERRQ(ierr);
for (i=0; i<baij->B->cmap->n/bs; i++) {
for (j=0; j<bs; j++) {
garray[i*bs+j] = bs*baij->garray[i]+j;
}
}
ierr = VecCreateGhost(PetscObjectComm((PetscObject)mat),mat->cmap->n,PETSC_DETERMINE,B->cmap->n,garray,&c->vscale);CHKERRQ(ierr);
ierr = PetscFree(garray);CHKERRQ(ierr);
}
} else {
Mat_MPIAIJ *aij=(Mat_MPIAIJ*)mat->data;
Mat_SeqAIJ *spA,*spB;
A = aij->A; spA = (Mat_SeqAIJ*)A->data; A_val = spA->a;
B = aij->B; spB = (Mat_SeqAIJ*)B->data; B_val = spB->a;
nz = spA->nz + spB->nz; /* total nonzero entries of mat */
if (!aij->colmap) {
/* Allow access to data structures of local part of matrix
- creates aij->colmap which maps global column number to local number in part B */
ierr = MatCreateColmap_MPIAIJ_Private(mat);CHKERRQ(ierr);
colmap = aij->colmap;
}
ierr = MatGetColumnIJ_SeqAIJ_Color(A,0,PETSC_FALSE,PETSC_FALSE,&ncols,&A_ci,&A_cj,&spidxA,NULL);CHKERRQ(ierr);
ierr = MatGetColumnIJ_SeqAIJ_Color(B,0,PETSC_FALSE,PETSC_FALSE,&ncols,&B_ci,&B_cj,&spidxB,NULL);CHKERRQ(ierr);
bs = 1; /* only bs=1 is supported for non MPIBAIJ matrix */
if (ctype == IS_COLORING_GLOBAL && c->htype[0] == 'd') { /* create vscale for storing dx */
ierr = VecCreateGhost(PetscObjectComm((PetscObject)mat),mat->cmap->n,PETSC_DETERMINE,B->cmap->n,aij->garray,&c->vscale);CHKERRQ(ierr);
}
}
m = mat->rmap->n/bs;
cstart = mat->cmap->rstart/bs;
cend = mat->cmap->rend/bs;
ierr = PetscMalloc1(nis,&c->ncolumns);CHKERRQ(ierr);
ierr = PetscMalloc1(nis,&c->columns);CHKERRQ(ierr);
ierr = PetscMalloc1(nis,&c->nrows);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)c,3*nis*sizeof(PetscInt));CHKERRQ(ierr);
if (c->htype[0] == 'd') {
ierr = PetscMalloc1(nz,&Jentry);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)c,nz*sizeof(MatEntry));CHKERRQ(ierr);
c->matentry = Jentry;
} else if (c->htype[0] == 'w') {
ierr = PetscMalloc1(nz,&Jentry2);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)c,nz*sizeof(MatEntry2));CHKERRQ(ierr);
c->matentry2 = Jentry2;
} else SETERRQ(PetscObjectComm((PetscObject)mat),PETSC_ERR_SUP,"htype is not supported");
ierr = PetscMalloc2(m+1,&rowhit,m+1,&valaddrhit);CHKERRQ(ierr);
nz = 0;
ierr = ISColoringGetIS(iscoloring,PETSC_IGNORE,&isa);CHKERRQ(ierr);
for (i=0; i<nis; i++) { /* for each local color */
ierr = ISGetLocalSize(isa[i],&n);CHKERRQ(ierr);
ierr = ISGetIndices(isa[i],&is);CHKERRQ(ierr);
c->ncolumns[i] = n; /* local number of columns of this color on this process */
if (n) {
ierr = PetscMalloc1(n,&c->columns[i]);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)c,n*sizeof(PetscInt));CHKERRQ(ierr);
ierr = PetscMemcpy(c->columns[i],is,n*sizeof(PetscInt));CHKERRQ(ierr);
} else {
c->columns[i] = 0;
}
if (ctype == IS_COLORING_GLOBAL) {
/* Determine nctot, the total (parallel) number of columns of this color */
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)mat),&size);CHKERRQ(ierr);
ierr = PetscMalloc2(size,&ncolsonproc,size,&disp);CHKERRQ(ierr);
/* ncolsonproc[j]: local ncolumns on proc[j] of this color */
ierr = PetscMPIIntCast(n,&nn);CHKERRQ(ierr);
ierr = MPI_Allgather(&nn,1,MPI_INT,ncolsonproc,1,MPI_INT,PetscObjectComm((PetscObject)mat));CHKERRQ(ierr);
nctot = 0; for (j=0; j<size; j++) nctot += ncolsonproc[j];
if (!nctot) {
ierr = PetscInfo(mat,"Coloring of matrix has some unneeded colors with no corresponding rows\n");CHKERRQ(ierr);
}
disp[0] = 0;
for (j=1; j<size; j++) {
disp[j] = disp[j-1] + ncolsonproc[j-1];
}
/* Get cols, the complete list of columns for this color on each process */
ierr = PetscMalloc1(nctot+1,&cols);CHKERRQ(ierr);
ierr = MPI_Allgatherv((void*)is,n,MPIU_INT,cols,ncolsonproc,disp,MPIU_INT,PetscObjectComm((PetscObject)mat));CHKERRQ(ierr);
ierr = PetscFree2(ncolsonproc,disp);CHKERRQ(ierr);
} else if (ctype == IS_COLORING_GHOSTED) {
/* Determine local number of columns of this color on this process, including ghost points */
nctot = n;
ierr = PetscMalloc1(nctot+1,&cols);CHKERRQ(ierr);
ierr = PetscMemcpy(cols,is,n*sizeof(PetscInt));CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Not provided for this MatFDColoring type");
/* Mark all rows affect by these columns */
ierr = PetscMemzero(rowhit,m*sizeof(PetscInt));CHKERRQ(ierr);
bs2 = bs*bs;
nrows_i = 0;
for (j=0; j<nctot; j++) { /* loop over columns*/
if (ctype == IS_COLORING_GHOSTED) {
col = ltog[cols[j]];
} else {
col = cols[j];
}
if (col >= cstart && col < cend) { /* column is in A, diagonal block of mat */
row = A_cj + A_ci[col-cstart];
nrows = A_ci[col-cstart+1] - A_ci[col-cstart];
nrows_i += nrows;
/* loop over columns of A marking them in rowhit */
for (k=0; k<nrows; k++) {
/* set valaddrhit for part A */
spidx = bs2*spidxA[A_ci[col-cstart] + k];
valaddrhit[*row] = &A_val[spidx];
rowhit[*row++] = col - cstart + 1; /* local column index */
}
} else { /* column is in B, off-diagonal block of mat */
#if defined(PETSC_USE_CTABLE)
ierr = PetscTableFind(colmap,col+1,&colb);CHKERRQ(ierr);
colb--;
#else
colb = colmap[col] - 1; /* local column index */
#endif
if (colb == -1) {
nrows = 0;
} else {
colb = colb/bs;
row = B_cj + B_ci[colb];
nrows = B_ci[colb+1] - B_ci[colb];
}
nrows_i += nrows;
/* loop over columns of B marking them in rowhit */
for (k=0; k<nrows; k++) {
/* set valaddrhit for part B */
spidx = bs2*spidxB[B_ci[colb] + k];
valaddrhit[*row] = &B_val[spidx];
rowhit[*row++] = colb + 1 + cend - cstart; /* local column index */
}
}
}
c->nrows[i] = nrows_i;
if (c->htype[0] == 'd') {
for (j=0; j<m; j++) {
if (rowhit[j]) {
Jentry[nz].row = j; /* local row index */
Jentry[nz].col = rowhit[j] - 1; /* local column index */
Jentry[nz].valaddr = valaddrhit[j]; /* address of mat value for this entry */
nz++;
}
}
} else { /* c->htype == 'wp' */
for (j=0; j<m; j++) {
if (rowhit[j]) {
Jentry2[nz].row = j; /* local row index */
Jentry2[nz].valaddr = valaddrhit[j]; /* address of mat value for this entry */
nz++;
}
}
}
ierr = PetscFree(cols);CHKERRQ(ierr);
}
if (bcols > 1) { /* reorder Jentry for faster MatFDColoringApply() */
ierr = MatFDColoringSetUpBlocked_AIJ_Private(mat,c,nz);CHKERRQ(ierr);
}
if (isBAIJ) {
ierr = MatRestoreColumnIJ_SeqBAIJ_Color(A,0,PETSC_FALSE,PETSC_FALSE,&ncols,&A_ci,&A_cj,&spidxA,NULL);CHKERRQ(ierr);
ierr = MatRestoreColumnIJ_SeqBAIJ_Color(B,0,PETSC_FALSE,PETSC_FALSE,&ncols,&B_ci,&B_cj,&spidxB,NULL);CHKERRQ(ierr);
ierr = PetscMalloc1(bs*mat->rmap->n,&c->dy);CHKERRQ(ierr);
} else {
ierr = MatRestoreColumnIJ_SeqAIJ_Color(A,0,PETSC_FALSE,PETSC_FALSE,&ncols,&A_ci,&A_cj,&spidxA,NULL);CHKERRQ(ierr);
ierr = MatRestoreColumnIJ_SeqAIJ_Color(B,0,PETSC_FALSE,PETSC_FALSE,&ncols,&B_ci,&B_cj,&spidxB,NULL);CHKERRQ(ierr);
}
ierr = ISColoringRestoreIS(iscoloring,&isa);CHKERRQ(ierr);
ierr = PetscFree2(rowhit,valaddrhit);CHKERRQ(ierr);
if (ctype == IS_COLORING_GHOSTED) {
ierr = ISLocalToGlobalMappingRestoreIndices(map,<og);CHKERRQ(ierr);
}
ierr = PetscInfo3(c,"ncolors %D, brows %D and bcols %D are used.\n",c->ncolors,c->brows,c->bcols);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMView_DA_2d(DM da,PetscViewer viewer)
{
PetscErrorCode ierr;
PetscMPIInt rank;
PetscBool iascii,isdraw,isbinary;
DM_DA *dd = (DM_DA*)da->data;
#if defined(PETSC_HAVE_MATLAB_ENGINE)
PetscBool ismatlab;
#endif
PetscFunctionBegin;
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)da),&rank);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERBINARY,&isbinary);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MATLAB_ENGINE)
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERMATLAB,&ismatlab);CHKERRQ(ierr);
#endif
if (iascii) {
PetscViewerFormat format;
ierr = PetscViewerGetFormat(viewer, &format);CHKERRQ(ierr);
if (format != PETSC_VIEWER_ASCII_VTK && format != PETSC_VIEWER_ASCII_VTK_CELL) {
DMDALocalInfo info;
ierr = DMDAGetLocalInfo(da,&info);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"Processor [%d] M %D N %D m %D n %D w %D s %D\n",rank,dd->M,dd->N,dd->m,dd->n,dd->w,dd->s);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(viewer,"X range of indices: %D %D, Y range of indices: %D %D\n",info.xs,info.xs+info.xm,info.ys,info.ys+info.ym);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);CHKERRQ(ierr);
} else {
ierr = DMView_DA_VTK(da,viewer);CHKERRQ(ierr);
}
} else if (isdraw) {
PetscDraw draw;
double ymin = -1*dd->s-1,ymax = dd->N+dd->s;
double xmin = -1*dd->s-1,xmax = dd->M+dd->s;
double x,y;
PetscInt base;
const PetscInt *idx;
char node[10];
PetscBool isnull;
ierr = PetscViewerDrawGetDraw(viewer,0,&draw);CHKERRQ(ierr);
ierr = PetscDrawIsNull(draw,&isnull);CHKERRQ(ierr); if (isnull) PetscFunctionReturn(0);
if (!da->coordinates) {
ierr = PetscDrawSetCoordinates(draw,xmin,ymin,xmax,ymax);CHKERRQ(ierr);
}
ierr = PetscDrawSynchronizedClear(draw);CHKERRQ(ierr);
/* first processor draw all node lines */
if (!rank) {
ymin = 0.0; ymax = dd->N - 1;
for (xmin=0; xmin<dd->M; xmin++) {
ierr = PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_BLACK);CHKERRQ(ierr);
}
xmin = 0.0; xmax = dd->M - 1;
for (ymin=0; ymin<dd->N; ymin++) {
ierr = PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_BLACK);CHKERRQ(ierr);
}
}
ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr);
ierr = PetscDrawPause(draw);CHKERRQ(ierr);
/* draw my box */
ymin = dd->ys; ymax = dd->ye - 1; xmin = dd->xs/dd->w;
xmax =(dd->xe-1)/dd->w;
ierr = PetscDrawLine(draw,xmin,ymin,xmax,ymin,PETSC_DRAW_RED);CHKERRQ(ierr);
ierr = PetscDrawLine(draw,xmin,ymin,xmin,ymax,PETSC_DRAW_RED);CHKERRQ(ierr);
ierr = PetscDrawLine(draw,xmin,ymax,xmax,ymax,PETSC_DRAW_RED);CHKERRQ(ierr);
ierr = PetscDrawLine(draw,xmax,ymin,xmax,ymax,PETSC_DRAW_RED);CHKERRQ(ierr);
/* put in numbers */
base = (dd->base)/dd->w;
for (y=ymin; y<=ymax; y++) {
for (x=xmin; x<=xmax; x++) {
sprintf(node,"%d",(int)base++);
ierr = PetscDrawString(draw,x,y,PETSC_DRAW_BLACK,node);CHKERRQ(ierr);
}
}
ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr);
ierr = PetscDrawPause(draw);CHKERRQ(ierr);
/* overlay ghost numbers, useful for error checking */
/* put in numbers */
base = 0;
ierr = ISLocalToGlobalMappingGetIndices(da->ltogmapb,&idx);CHKERRQ(ierr);
ymin = dd->Ys; ymax = dd->Ye; xmin = dd->Xs; xmax = dd->Xe;
for (y=ymin; y<ymax; y++) {
for (x=xmin; x<xmax; x++) {
if ((base % dd->w) == 0) {
sprintf(node,"%d",(int)(idx[base]));
ierr = PetscDrawString(draw,x/dd->w,y,PETSC_DRAW_BLUE,node);CHKERRQ(ierr);
}
base++;
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(da->ltogmapb,&idx);
ierr = PetscDrawSynchronizedFlush(draw);CHKERRQ(ierr);
ierr = PetscDrawPause(draw);CHKERRQ(ierr);
} else if (isbinary) {
ierr = DMView_DA_Binary(da,viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MATLAB_ENGINE)
} else if (ismatlab) {
ierr = DMView_DA_Matlab(da,viewer);CHKERRQ(ierr);
#endif
}
PetscFunctionReturn(0);
}
static PetscErrorCode MatSetupDM_HYPRESStruct(Mat mat,DM da)
{
PetscErrorCode ierr;
Mat_HYPRESStruct *ex = (Mat_HYPRESStruct*) mat->data;
PetscInt dim,dof,sw[3],nx,ny,nz;
PetscInt ilower[3],iupper[3],ssize,i;
DMBoundaryType px,py,pz;
DMDAStencilType st;
PetscInt nparts= 1; /* assuming only one part */
PetscInt part = 0;
ISLocalToGlobalMapping ltog;
PetscFunctionBegin;
ex->da = da;
ierr = PetscObjectReference((PetscObject)da);CHKERRQ(ierr);
ierr = DMDAGetInfo(ex->da,&dim,0,0,0,0,0,0,&dof,&sw[0],&px,&py,&pz,&st);CHKERRQ(ierr);
ierr = DMDAGetCorners(ex->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);CHKERRQ(ierr);
iupper[0] += ilower[0] - 1;
iupper[1] += ilower[1] - 1;
iupper[2] += ilower[2] - 1;
/* the hypre_Box is used to zero out the matrix entries in MatZeroValues() */
ex->hbox.imin[0] = ilower[0];
ex->hbox.imin[1] = ilower[1];
ex->hbox.imin[2] = ilower[2];
ex->hbox.imax[0] = iupper[0];
ex->hbox.imax[1] = iupper[1];
ex->hbox.imax[2] = iupper[2];
ex->dofs_order = 0;
/* assuming that the same number of dofs on each gridpoint. Also assume all cell-centred based */
ex->nvars= dof;
/* create the hypre grid object and set its information */
if (px || py || pz) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add periodic support by calling HYPRE_SStructGridSetPeriodic()");
PetscStackCallStandard(HYPRE_SStructGridCreate,(ex->hcomm,dim,nparts,&ex->ss_grid));
PetscStackCallStandard(HYPRE_SStructGridSetExtents,(ex->ss_grid,part,ex->hbox.imin,ex->hbox.imax));
{
HYPRE_SStructVariable *vartypes;
ierr = PetscMalloc1(ex->nvars,&vartypes);CHKERRQ(ierr);
for (i= 0; i< ex->nvars; i++) vartypes[i]= HYPRE_SSTRUCT_VARIABLE_CELL;
PetscStackCallStandard(HYPRE_SStructGridSetVariables,(ex->ss_grid, part, ex->nvars,vartypes));
ierr = PetscFree(vartypes);CHKERRQ(ierr);
}
PetscStackCallStandard(HYPRE_SStructGridAssemble,(ex->ss_grid));
sw[1] = sw[0];
sw[2] = sw[1];
/* PetscStackCallStandard(HYPRE_SStructGridSetNumGhost,(ex->ss_grid,sw)); */
/* create the hypre stencil object and set its information */
if (sw[0] > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for wider stencils");
if (st == DMDA_STENCIL_BOX) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for box stencils");
if (dim == 1) {
PetscInt offsets[3][1] = {{-1},{0},{1}};
PetscInt j, cnt;
ssize = 3*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i = 0; i < (ex->nvars); i++) {
for (j = 0; j < 3; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
} else if (dim == 2) {
PetscInt offsets[5][2] = {{0,-1},{-1,0},{0,0},{1,0},{0,1}};
PetscInt j, cnt;
ssize = 5*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i= 0; i< (ex->nvars); i++) {
for (j= 0; j< 5; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
} else if (dim == 3) {
PetscInt offsets[7][3] = {{0,0,-1},{0,-1,0},{-1,0,0},{0,0,0},{1,0,0},{0,1,0},{0,0,1}};
PetscInt j, cnt;
ssize = 7*(ex->nvars);
PetscStackCallStandard(HYPRE_SStructStencilCreate,(dim,ssize,&ex->ss_stencil));
cnt= 0;
for (i= 0; i< (ex->nvars); i++) {
for (j= 0; j< 7; j++) {
PetscStackCallStandard(HYPRE_SStructStencilSetEntry,(ex->ss_stencil, cnt, (HYPRE_Int *)offsets[j], i));
cnt++;
}
}
}
/* create the HYPRE graph */
PetscStackCallStandard(HYPRE_SStructGraphCreate,(ex->hcomm, ex->ss_grid, &(ex->ss_graph)));
/* set the stencil graph. Note that each variable has the same graph. This means that each
variable couples to all the other variable and with the same stencil pattern. */
for (i= 0; i< (ex->nvars); i++) {
PetscStackCallStandard(HYPRE_SStructGraphSetStencil,(ex->ss_graph,part,i,ex->ss_stencil));
}
PetscStackCallStandard(HYPRE_SStructGraphAssemble,(ex->ss_graph));
/* create the HYPRE sstruct vectors for rhs and solution */
PetscStackCallStandard(HYPRE_SStructVectorCreate,(ex->hcomm,ex->ss_grid,&ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorCreate,(ex->hcomm,ex->ss_grid,&ex->ss_x));
PetscStackCallStandard(HYPRE_SStructVectorInitialize,(ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorInitialize,(ex->ss_x));
PetscStackCallStandard(HYPRE_SStructVectorAssemble,(ex->ss_b));
PetscStackCallStandard(HYPRE_SStructVectorAssemble,(ex->ss_x));
/* create the hypre matrix object and set its information */
PetscStackCallStandard(HYPRE_SStructMatrixCreate,(ex->hcomm,ex->ss_graph,&ex->ss_mat));
PetscStackCallStandard(HYPRE_SStructGridDestroy,(ex->ss_grid));
PetscStackCallStandard(HYPRE_SStructStencilDestroy,(ex->ss_stencil));
if (ex->needsinitialization) {
PetscStackCallStandard(HYPRE_SStructMatrixInitialize,(ex->ss_mat));
ex->needsinitialization = PETSC_FALSE;
}
/* set the global and local sizes of the matrix */
ierr = DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);CHKERRQ(ierr);
ierr = MatSetSizes(mat,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->rmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->cmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
if (dim == 3) {
mat->ops->setvalueslocal = MatSetValuesLocal_HYPRESStruct_3d;
mat->ops->zerorowslocal = MatZeroRowsLocal_HYPRESStruct_3d;
mat->ops->zeroentries = MatZeroEntries_HYPRESStruct_3d;
ierr = MatZeroEntries_HYPRESStruct_3d(mat);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only support for 3d DMDA currently");
/* get values that will be used repeatedly in MatSetValuesLocal() and MatZeroRowsLocal() repeatedly */
ierr = MatGetOwnershipRange(mat,&ex->rstart,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(ex->da,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog, (const PetscInt **) &ex->gindices);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(ex->da,0,0,0,&ex->gnx,&ex->gnxgny,&ex->gnxgnygnz);CHKERRQ(ierr);
ex->gnxgny *= ex->gnx;
ex->gnxgnygnz *= ex->gnxgny;
ierr = DMDAGetCorners(ex->da,&ex->xs,&ex->ys,&ex->zs,&ex->nx,&ex->ny,&ex->nz);CHKERRQ(ierr);
ex->nxny = ex->nx*ex->ny;
ex->nxnynz = ex->nz*ex->nxny;
PetscFunctionReturn(0);
}
static PetscErrorCode MatSetupDM_HYPREStruct(Mat mat,DM da)
{
PetscErrorCode ierr;
Mat_HYPREStruct *ex = (Mat_HYPREStruct*) mat->data;
PetscInt dim,dof,sw[3],nx,ny,nz,ilower[3],iupper[3],ssize,i;
DMBoundaryType px,py,pz;
DMDAStencilType st;
ISLocalToGlobalMapping ltog;
PetscFunctionBegin;
ex->da = da;
ierr = PetscObjectReference((PetscObject)da);CHKERRQ(ierr);
ierr = DMDAGetInfo(ex->da,&dim,0,0,0,0,0,0,&dof,&sw[0],&px,&py,&pz,&st);CHKERRQ(ierr);
ierr = DMDAGetCorners(ex->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);CHKERRQ(ierr);
iupper[0] += ilower[0] - 1;
iupper[1] += ilower[1] - 1;
iupper[2] += ilower[2] - 1;
/* the hypre_Box is used to zero out the matrix entries in MatZeroValues() */
ex->hbox.imin[0] = ilower[0];
ex->hbox.imin[1] = ilower[1];
ex->hbox.imin[2] = ilower[2];
ex->hbox.imax[0] = iupper[0];
ex->hbox.imax[1] = iupper[1];
ex->hbox.imax[2] = iupper[2];
/* create the hypre grid object and set its information */
if (dof > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Currently only support for scalar problems");
if (px || py || pz) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add periodic support by calling HYPRE_StructGridSetPeriodic()");
PetscStackCallStandard(HYPRE_StructGridCreate,(ex->hcomm,dim,&ex->hgrid));
PetscStackCallStandard(HYPRE_StructGridSetExtents,(ex->hgrid,(HYPRE_Int *)ilower,(HYPRE_Int *)iupper));
PetscStackCallStandard(HYPRE_StructGridAssemble,(ex->hgrid));
sw[1] = sw[0];
sw[2] = sw[1];
PetscStackCallStandard(HYPRE_StructGridSetNumGhost,(ex->hgrid,(HYPRE_Int *)sw));
/* create the hypre stencil object and set its information */
if (sw[0] > 1) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for wider stencils");
if (st == DMDA_STENCIL_BOX) SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Ask us to add support for box stencils");
if (dim == 1) {
PetscInt offsets[3][1] = {{-1},{0},{1}};
ssize = 3;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
} else if (dim == 2) {
PetscInt offsets[5][2] = {{0,-1},{-1,0},{0,0},{1,0},{0,1}};
ssize = 5;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
} else if (dim == 3) {
PetscInt offsets[7][3] = {{0,0,-1},{0,-1,0},{-1,0,0},{0,0,0},{1,0,0},{0,1,0},{0,0,1}};
ssize = 7;
PetscStackCallStandard(HYPRE_StructStencilCreate,(dim,ssize,&ex->hstencil));
for (i=0; i<ssize; i++) {
PetscStackCallStandard(HYPRE_StructStencilSetElement,(ex->hstencil,i,(HYPRE_Int *)offsets[i]));
}
}
/* create the HYPRE vector for rhs and solution */
PetscStackCallStandard(HYPRE_StructVectorCreate,(ex->hcomm,ex->hgrid,&ex->hb));
PetscStackCallStandard(HYPRE_StructVectorCreate,(ex->hcomm,ex->hgrid,&ex->hx));
PetscStackCallStandard(HYPRE_StructVectorInitialize,(ex->hb));
PetscStackCallStandard(HYPRE_StructVectorInitialize,(ex->hx));
PetscStackCallStandard(HYPRE_StructVectorAssemble,(ex->hb));
PetscStackCallStandard(HYPRE_StructVectorAssemble,(ex->hx));
/* create the hypre matrix object and set its information */
PetscStackCallStandard(HYPRE_StructMatrixCreate,(ex->hcomm,ex->hgrid,ex->hstencil,&ex->hmat));
PetscStackCallStandard(HYPRE_StructGridDestroy,(ex->hgrid));
PetscStackCallStandard(HYPRE_StructStencilDestroy,(ex->hstencil));
if (ex->needsinitialization) {
PetscStackCallStandard(HYPRE_StructMatrixInitialize,(ex->hmat));
ex->needsinitialization = PETSC_FALSE;
}
/* set the global and local sizes of the matrix */
ierr = DMDAGetCorners(da,0,0,0,&nx,&ny,&nz);CHKERRQ(ierr);
ierr = MatSetSizes(mat,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->rmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(mat->cmap,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(mat->cmap);CHKERRQ(ierr);
if (dim == 3) {
mat->ops->setvalueslocal = MatSetValuesLocal_HYPREStruct_3d;
mat->ops->zerorowslocal = MatZeroRowsLocal_HYPREStruct_3d;
mat->ops->zeroentries = MatZeroEntries_HYPREStruct_3d;
ierr = MatZeroEntries_HYPREStruct_3d(mat);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)da),PETSC_ERR_SUP,"Only support for 3d DMDA currently");
/* get values that will be used repeatedly in MatSetValuesLocal() and MatZeroRowsLocal() repeatedly */
ierr = MatGetOwnershipRange(mat,&ex->rstart,NULL);CHKERRQ(ierr);
ierr = DMGetLocalToGlobalMapping(ex->da,<og);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(ltog, (const PetscInt **) &ex->gindices);CHKERRQ(ierr);
ierr = DMDAGetGhostCorners(ex->da,0,0,0,&ex->gnx,&ex->gnxgny,0);CHKERRQ(ierr);
ex->gnxgny *= ex->gnx;
ierr = DMDAGetCorners(ex->da,&ex->xs,&ex->ys,&ex->zs,&ex->nx,&ex->ny,0);CHKERRQ(ierr);
ex->nxny = ex->nx*ex->ny;
PetscFunctionReturn(0);
}
