PetscErrorCode DMSetUp_Composite(DM dm)
{
PetscErrorCode ierr;
PetscInt nprev = 0;
PetscMPIInt rank,size;
DM_Composite *com = (DM_Composite*)dm->data;
struct DMCompositeLink *next = com->next;
PetscLayout map;
PetscFunctionBegin;
if (com->setup) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Packer has already been setup");
ierr = PetscLayoutCreate(((PetscObject)dm)->comm,&map);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(map,com->n);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(map,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(map,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(map);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(map,&com->N);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(map,&com->rstart,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&map);CHKERRQ(ierr);
/* now set the rstart for each linked vector */
ierr = MPI_Comm_rank(((PetscObject)dm)->comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)dm)->comm,&size);CHKERRQ(ierr);
while (next) {
next->rstart = nprev;
nprev += next->n;
next->grstart = com->rstart + next->rstart;
ierr = PetscMalloc(size*sizeof(PetscInt),&next->grstarts);CHKERRQ(ierr);
ierr = MPI_Allgather(&next->grstart,1,MPIU_INT,next->grstarts,1,MPIU_INT,((PetscObject)dm)->comm);CHKERRQ(ierr);
next = next->next;
}
com->setup = PETSC_TRUE;
PetscFunctionReturn(0);
}
PetscErrorCode ISView_General_HDF5(IS is, PetscViewer viewer)
{
hid_t filespace; /* file dataspace identifier */
hid_t chunkspace; /* chunk dataset property identifier */
hid_t plist_id; /* property list identifier */
hid_t dset_id; /* dataset identifier */
hid_t memspace; /* memory dataspace identifier */
hid_t inttype; /* int type (H5T_NATIVE_INT or H5T_NATIVE_LLONG) */
hid_t file_id, group;
herr_t status;
hsize_t dim, maxDims[3], dims[3], chunkDims[3], count[3],offset[3];
PetscInt bs, N, n, timestep, low;
const PetscInt *ind;
const char *isname;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = ISGetBlockSize(is,&bs);CHKERRQ(ierr);
ierr = PetscViewerHDF5OpenGroup(viewer, &file_id, &group);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetTimestep(viewer, ×tep);CHKERRQ(ierr);
/* Create the dataspace for the dataset.
*
* dims - holds the current dimensions of the dataset
*
* maxDims - holds the maximum dimensions of the dataset (unlimited
* for the number of time steps with the current dimensions for the
* other dimensions; so only additional time steps can be added).
*
* chunkDims - holds the size of a single time step (required to
* permit extending dataset).
*/
dim = 0;
if (timestep >= 0) {
dims[dim] = timestep+1;
maxDims[dim] = H5S_UNLIMITED;
chunkDims[dim] = 1;
++dim;
}
ierr = ISGetSize(is, &N);CHKERRQ(ierr);
ierr = ISGetLocalSize(is, &n);CHKERRQ(ierr);
ierr = PetscHDF5IntCast(N/bs,dims + dim);CHKERRQ(ierr);
maxDims[dim] = dims[dim];
chunkDims[dim] = dims[dim];
++dim;
if (bs >= 1) {
dims[dim] = bs;
maxDims[dim] = dims[dim];
chunkDims[dim] = dims[dim];
++dim;
}
filespace = H5Screate_simple(dim, dims, maxDims);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Cannot H5Screate_simple()");
#if defined(PETSC_USE_64BIT_INDICES)
inttype = H5T_NATIVE_LLONG;
#else
inttype = H5T_NATIVE_INT;
#endif
/* Create the dataset with default properties and close filespace */
ierr = PetscObjectGetName((PetscObject) is, &isname);CHKERRQ(ierr);
if (!H5Lexists(group, isname, H5P_DEFAULT)) {
/* Create chunk */
chunkspace = H5Pcreate(H5P_DATASET_CREATE);
if (chunkspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Pcreate()");
status = H5Pset_chunk(chunkspace, dim, chunkDims);CHKERRQ(status);
#if (H5_VERS_MAJOR * 10000 + H5_VERS_MINOR * 100 + H5_VERS_RELEASE >= 10800)
dset_id = H5Dcreate2(group, isname, inttype, filespace, H5P_DEFAULT, chunkspace, H5P_DEFAULT);
#else
dset_id = H5Dcreate(group, isname, inttype, filespace, H5P_DEFAULT);
#endif
if (dset_id == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Dcreate2()");
status = H5Pclose(chunkspace);CHKERRQ(status);
} else {
dset_id = H5Dopen2(group, isname, H5P_DEFAULT);
status = H5Dset_extent(dset_id, dims);CHKERRQ(status);
}
status = H5Sclose(filespace);CHKERRQ(status);
/* Each process defines a dataset and writes it to the hyperslab in the file */
dim = 0;
if (timestep >= 0) {
count[dim] = 1;
++dim;
}
ierr = PetscHDF5IntCast(n/bs,count + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
count[dim] = bs;
++dim;
}
if (n > 0) {
memspace = H5Screate_simple(dim, count, NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate_simple()");
} else {
/* Can't create dataspace with zero for any dimension, so create null dataspace. */
memspace = H5Screate(H5S_NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate()");
}
/* Select hyperslab in the file */
ierr = PetscLayoutGetRange(is->map, &low, NULL);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
offset[dim] = timestep;
++dim;
}
ierr = PetscHDF5IntCast(low/bs,offset + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
offset[dim] = 0;
++dim;
}
if (n > 0) {
filespace = H5Dget_space(dset_id);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Dget_space()");
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL);CHKERRQ(status);
} else {
/* Create null filespace to match null memspace. */
filespace = H5Screate(H5S_NULL);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Screate(H5S_NULL)");
}
/* Create property list for collective dataset write */
plist_id = H5Pcreate(H5P_DATASET_XFER);
if (plist_id == -1) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "Cannot H5Pcreate()");
#if defined(PETSC_HAVE_H5PSET_FAPL_MPIO)
status = H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);CHKERRQ(status);
#endif
/* To write dataset independently use H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT) */
ierr = ISGetIndices(is, &ind);CHKERRQ(ierr);
status = H5Dwrite(dset_id, inttype, memspace, filespace, plist_id, ind);CHKERRQ(status);
status = H5Fflush(file_id, H5F_SCOPE_GLOBAL);CHKERRQ(status);
ierr = ISGetIndices(is, &ind);CHKERRQ(ierr);
/* Close/release resources */
if (group != file_id) {status = H5Gclose(group);CHKERRQ(status);}
status = H5Pclose(plist_id);CHKERRQ(status);
status = H5Sclose(filespace);CHKERRQ(status);
status = H5Sclose(memspace);CHKERRQ(status);
status = H5Dclose(dset_id);CHKERRQ(status);
ierr = PetscInfo1(is, "Wrote IS object with name %s\n", isname);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMPlexPreallocateOperator(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix)
{
MPI_Comm comm;
MatType mtype;
PetscSF sf, sfDof, sfAdj;
PetscSection leafSectionAdj, rootSectionAdj, sectionAdj, anchorSectionAdj;
PetscInt nroots, nleaves, l, p;
const PetscInt *leaves;
const PetscSFNode *remotes;
PetscInt dim, pStart, pEnd, numDof, globalOffStart, globalOffEnd, numCols;
PetscInt *tmpAdj = NULL, *adj, *rootAdj, *anchorAdj = NULL, *cols, *remoteOffsets;
PetscInt adjSize;
PetscLayout rLayout;
PetscInt locRows, rStart, rEnd, r;
PetscMPIInt size;
PetscBool doCommLocal, doComm, debug = PETSC_FALSE, isSymBlock, isSymSeqBlock, isSymMPIBlock;
PetscBool useAnchors;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 3);
PetscValidHeaderSpecific(sectionGlobal, PETSC_SECTION_CLASSID, 4);
PetscValidHeaderSpecific(A, MAT_CLASSID, 9);
if (dnz) PetscValidPointer(dnz,5);
if (onz) PetscValidPointer(onz,6);
if (dnzu) PetscValidPointer(dnzu,7);
if (onzu) PetscValidPointer(onzu,8);
ierr = PetscLogEventBegin(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL, "-dm_view_preallocation", &debug, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
doCommLocal = (size > 1) && (nroots >= 0) ? PETSC_TRUE : PETSC_FALSE;
ierr = MPI_Allreduce(&doCommLocal, &doComm, 1, MPIU_BOOL, MPI_LAND, comm);CHKERRQ(ierr);
/* Create dof SF based on point SF */
if (debug) {
ierr = PetscPrintf(comm, "Input Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input Global Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionGlobal, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sf, NULL);CHKERRQ(ierr);
}
ierr = PetscSFCreateRemoteOffsets(sf, section, section, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, section, remoteOffsets, section, &sfDof);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Dof SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfDof, NULL);CHKERRQ(ierr);
}
/* Create section for dof adjacency (dof ==> # adj dof) */
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(section, &numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(leafSectionAdj, 0, numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSectionAdj, 0, numDof);CHKERRQ(ierr);
/* Fill in the ghost dofs on the interface */
ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, &remotes);CHKERRQ(ierr);
/* use constraints in finding adjacency in this routine */
ierr = DMPlexGetAdjacencyUseAnchors(dm,&useAnchors);CHKERRQ(ierr);
ierr = DMPlexSetAdjacencyUseAnchors(dm,PETSC_TRUE);CHKERRQ(ierr);
/*
section - maps points to (# dofs, local dofs)
sectionGlobal - maps points to (# dofs, global dofs)
leafSectionAdj - maps unowned local dofs to # adj dofs
rootSectionAdj - maps owned local dofs to # adj dofs
adj - adj global dofs indexed by leafSectionAdj
rootAdj - adj global dofs indexed by rootSectionAdj
sf - describes shared points across procs
sfDof - describes shared dofs across procs
sfAdj - describes shared adjacent dofs across procs
** The bootstrapping process involves six rounds with similar structure of visiting neighbors of each point.
(0). If there are point-to-point constraints, add the adjacencies of constrained points to anchors in anchorAdj
(This is done in DMPlexComputeAnchorAdjacencies())
1. Visit unowned points on interface, count adjacencies placing in leafSectionAdj
Reduce those counts to rootSectionAdj (now redundantly counting some interface points)
2. Visit owned points on interface, count adjacencies placing in rootSectionAdj
Create sfAdj connecting rootSectionAdj and leafSectionAdj
3. Visit unowned points on interface, write adjacencies to adj
Gather adj to rootAdj (note that there is redundancy in rootAdj when multiple procs find the same adjacencies)
4. Visit owned points on interface, write adjacencies to rootAdj
Remove redundancy in rootAdj
** The last two traversals use transitive closure
5. Visit all owned points in the subdomain, count dofs for each point (sectionAdj)
Allocate memory addressed by sectionAdj (cols)
6. Visit all owned points in the subdomain, insert dof adjacencies into cols
** Knowing all the column adjacencies, check ownership and sum into dnz and onz
*/
ierr = DMPlexComputeAnchorAdjacencies(dm,section,sectionGlobal,&anchorSectionAdj,&anchorAdj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation on Leaves:\n");CHKERRQ(ierr);
ierr = PetscSectionView(leafSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get maximum remote adjacency sizes for owned dofs on interface (roots) */
if (doComm) {
ierr = PetscSFReduceBegin(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
}
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Add in local adjacency sizes for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after local additions:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create adj SF based on dof SF */
ierr = PetscSFCreateRemoteOffsets(sfDof, rootSectionAdj, leafSectionAdj, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfDof, rootSectionAdj, remoteOffsets, leafSectionAdj, &sfAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfAdj, NULL);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfDof);CHKERRQ(ierr);
/* Create leaf adjacency */
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(leafSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscCalloc1(adjSize, &adj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof, anOff;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt aoff, i = 0;
ierr = PetscSectionGetOffset(leafSectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
adj[aoff+i] = (ngoff < 0 ? -(ngoff+1) : ngoff) + nd;
++i;
}
}
for (q = 0; q < anDof; q++) {
adj[aoff+i] = anchorAdj[anOff+q];
++i;
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Leaf adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, adj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Gather adjacenct indices to root */
ierr = PetscSectionGetStorageSize(rootSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize, &rootAdj);CHKERRQ(ierr);
for (r = 0; r < adjSize; ++r) rootAdj[r] = -1;
if (doComm) {
ierr = PetscSFGatherBegin(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
ierr = PetscSFGatherEnd(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfAdj);CHKERRQ(ierr);
ierr = PetscFree(adj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices after gather\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Add in local adjacency indices for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof, anOff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt adof, aoff, i;
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
i = adof-1;
for (q = 0; q < anDof; q++) {
rootAdj[aoff+i] = anchorAdj[anOff+q];
--i;
}
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
rootAdj[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
--i;
}
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Compress indices */
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, cdof, off, d;
PetscInt adof, aoff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
for (d = off; d < off+dof-cdof; ++d) {
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
ierr = PetscSortRemoveDupsInt(&adof, &rootAdj[aoff]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSectionAdj, d, adof);CHKERRQ(ierr);
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after compression:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Root adjacency indices after compression\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Build adjacency section: Maps global indices to sets of adjacent global indices */
ierr = PetscSectionGetOffsetRange(sectionGlobal, &globalOffStart, &globalOffEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, §ionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionAdj, globalOffStart, globalOffEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
ierr = PetscSectionSetDof(sectionAdj, goff+d, rdof);CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, noff;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, padj, &noff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(sectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get adjacent indices */
ierr = PetscSectionGetStorageSize(sectionAdj, &numCols);CHKERRQ(ierr);
ierr = PetscMalloc1(numCols, &cols);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof, anOff;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
PetscInt aoff, roff;
ierr = PetscSectionGetOffset(sectionAdj, goff+d, &aoff);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, off+d, &roff);CHKERRQ(ierr);
ierr = PetscMemcpy(&cols[aoff], &rootAdj[roff], rdof * sizeof(PetscInt));CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
PetscInt adof, aoff, i = 0;
ierr = PetscSectionGetDof(sectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
const PetscInt *ncind;
/* Adjacent points may not be in the section chart */
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintIndices(section, padj, &ncind);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd, ++i) {
cols[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
}
}
for (q = 0; q < anDof; q++, i++) {
cols[aoff+i] = anchorAdj[anOff + q];
}
if (i != adof) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of entries %D != %D for dof %D (point %D)", i, adof, d, p);
}
}
ierr = PetscSectionDestroy(&anchorSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSectionAdj);CHKERRQ(ierr);
ierr = PetscFree(anchorAdj);CHKERRQ(ierr);
ierr = PetscFree(rootAdj);CHKERRQ(ierr);
ierr = PetscFree(tmpAdj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Column indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, numCols, cols, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Create allocation vectors from adjacency graph */
ierr = MatGetLocalSize(A, &locRows, NULL);CHKERRQ(ierr);
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)A), &rLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr);
/* Only loop over blocks of rows */
if (rStart%bs || rEnd%bs) SETERRQ3(PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Invalid layout [%d, %d) for matrix, must be divisible by block size %d", rStart, rEnd, bs);
for (r = rStart/bs; r < rEnd/bs; ++r) {
const PetscInt row = r*bs;
PetscInt numCols, cStart, c;
ierr = PetscSectionGetDof(sectionAdj, row, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, row, &cStart);CHKERRQ(ierr);
for (c = cStart; c < cStart+numCols; ++c) {
if ((cols[c] >= rStart*bs) && (cols[c] < rEnd*bs)) {
++dnz[r-rStart];
if (cols[c] >= row) ++dnzu[r-rStart];
} else {
++onz[r-rStart];
if (cols[c] >= row) ++onzu[r-rStart];
}
}
}
if (bs > 1) {
for (r = 0; r < locRows/bs; ++r) {
dnz[r] /= bs;
onz[r] /= bs;
dnzu[r] /= bs;
onzu[r] /= bs;
}
}
/* Set matrix pattern */
ierr = MatXAIJSetPreallocation(A, bs, dnz, onz, dnzu, onzu);CHKERRQ(ierr);
ierr = MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
/* Check for symmetric storage */
ierr = MatGetType(A, &mtype);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
if (isSymBlock || isSymSeqBlock || isSymMPIBlock) {ierr = MatSetOption(A, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);CHKERRQ(ierr);}
/* Fill matrix with zeros */
if (fillMatrix) {
PetscScalar *values;
PetscInt maxRowLen = 0;
for (r = rStart; r < rEnd; ++r) {
PetscInt len;
ierr = PetscSectionGetDof(sectionAdj, r, &len);CHKERRQ(ierr);
maxRowLen = PetscMax(maxRowLen, len);
}
ierr = PetscCalloc1(maxRowLen, &values);CHKERRQ(ierr);
for (r = rStart; r < rEnd; ++r) {
PetscInt numCols, cStart;
ierr = PetscSectionGetDof(sectionAdj, r, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, r, &cStart);CHKERRQ(ierr);
ierr = MatSetValues(A, 1, &r, numCols, &cols[cStart], values, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree(values);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* restore original useAnchors */
ierr = DMPlexSetAdjacencyUseAnchors(dm,useAnchors);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionAdj);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
ISBuildTwoSided - Takes an IS that describes where we will go. Generates an IS that contains new numbers from remote or local
on the IS.
Collective on IS
Input Parameters
. to - an IS describes where we will go. Negative target rank will be ignored
. toindx - an IS describes what indices should send. NULL means sending natural numbering
Output Parameter:
. rows - contains new numbers from remote or local
Level: advanced
.seealso: MatPartitioningCreate(), ISPartitioningToNumbering(), ISPartitioningCount()
@*/
PetscErrorCode ISBuildTwoSided(IS ito,IS toindx, IS *rows)
{
const PetscInt *ito_indices,*toindx_indices;
PetscInt *send_indices,rstart,*recv_indices,nrecvs,nsends;
PetscInt *tosizes,*fromsizes,i,j,*tosizes_tmp,*tooffsets_tmp,ito_ln;
PetscMPIInt *toranks,*fromranks,size,target_rank,*fromperm_newtoold,nto,nfrom;
PetscLayout isrmap;
MPI_Comm comm;
PetscSF sf;
PetscSFNode *iremote;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)ito,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = ISGetLocalSize(ito,&ito_ln);CHKERRQ(ierr);
/* why we do not have ISGetLayout? */
isrmap = ito->map;
ierr = PetscLayoutGetRange(isrmap,&rstart,NULL);CHKERRQ(ierr);
ierr = ISGetIndices(ito,&ito_indices);CHKERRQ(ierr);
ierr = PetscCalloc2(size,&tosizes_tmp,size+1,&tooffsets_tmp);CHKERRQ(ierr);
for(i=0; i<ito_ln; i++){
if(ito_indices[i]<0) continue;
#if defined(PETSC_USE_DEBUG)
if(ito_indices[i]>=size) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"target rank %d is larger than communicator size %d ",ito_indices[i],size);
#endif
tosizes_tmp[ito_indices[i]]++;
}
nto = 0;
for(i=0; i<size; i++){
tooffsets_tmp[i+1] = tooffsets_tmp[i]+tosizes_tmp[i];
if(tosizes_tmp[i]>0) nto++;
}
ierr = PetscCalloc2(nto,&toranks,2*nto,&tosizes);CHKERRQ(ierr);
nto = 0;
for(i=0; i<size; i++){
if(tosizes_tmp[i]>0){
toranks[nto] = i;
tosizes[2*nto] = tosizes_tmp[i];/* size */
tosizes[2*nto+1] = tooffsets_tmp[i];/* offset */
nto++;
}
}
nsends = tooffsets_tmp[size];
ierr = PetscCalloc1(nsends,&send_indices);CHKERRQ(ierr);
if(toindx){
ierr = ISGetIndices(toindx,&toindx_indices);CHKERRQ(ierr);
}
for(i=0; i<ito_ln; i++){
if(ito_indices[i]<0) continue;
target_rank = ito_indices[i];
send_indices[tooffsets_tmp[target_rank]] = toindx? toindx_indices[i]:(i+rstart);
tooffsets_tmp[target_rank]++;
}
if(toindx){
ierr = ISRestoreIndices(toindx,&toindx_indices);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(ito,&ito_indices);CHKERRQ(ierr);
ierr = PetscFree2(tosizes_tmp,tooffsets_tmp);CHKERRQ(ierr);
ierr = PetscCommBuildTwoSided(comm,2,MPIU_INT,nto,toranks,tosizes,&nfrom,&fromranks,&fromsizes);CHKERRQ(ierr);
ierr = PetscFree2(toranks,tosizes);CHKERRQ(ierr);
ierr = PetscCalloc1(nfrom,&fromperm_newtoold);CHKERRQ(ierr);
for(i=0; i<nfrom; i++){
fromperm_newtoold[i] = i;
}
ierr = PetscSortMPIIntWithArray(nfrom,fromranks,fromperm_newtoold);CHKERRQ(ierr);
nrecvs = 0;
for(i=0; i<nfrom; i++){
nrecvs += fromsizes[i*2];
}
ierr = PetscCalloc1(nrecvs,&recv_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(nrecvs,&iremote);CHKERRQ(ierr);
nrecvs = 0;
for(i=0; i<nfrom; i++){
for(j=0; j<fromsizes[2*fromperm_newtoold[i]]; j++){
iremote[nrecvs].rank = fromranks[i];
iremote[nrecvs++].index = fromsizes[2*fromperm_newtoold[i]+1]+j;
}
}
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nsends,nrecvs,NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
/* how to put a prefix ? */
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,send_indices,recv_indices);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,send_indices,recv_indices);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree(fromranks);CHKERRQ(ierr);
ierr = PetscFree(fromsizes);CHKERRQ(ierr);
ierr = PetscFree(fromperm_newtoold);CHKERRQ(ierr);
ierr = PetscFree(send_indices);CHKERRQ(ierr);
if(rows){
ierr = PetscSortInt(nrecvs,recv_indices);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, nrecvs,recv_indices,PETSC_OWN_POINTER,rows);CHKERRQ(ierr);
}else{
ierr = PetscFree(recv_indices);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode ISView_General_Binary(IS is,PetscViewer viewer)
{
PetscErrorCode ierr;
PetscBool skipHeader,useMPIIO;
IS_General *isa = (IS_General*) is->data;
PetscMPIInt rank,size,mesgsize,tag = ((PetscObject)viewer)->tag, mesglen;
PetscInt n,N,len,j,tr[2];
int fdes;
MPI_Status status;
PetscInt message_count,flowcontrolcount,*values;
PetscFunctionBegin;
/* ierr = ISGetLayout(is,&map);CHKERRQ(ierr); */
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr);
tr[0] = IS_FILE_CLASSID;
tr[1] = N;
ierr = PetscViewerBinaryGetSkipHeader(viewer,&skipHeader);CHKERRQ(ierr);
if (!skipHeader) {
ierr = PetscViewerBinaryWrite(viewer,tr,2,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
}
ierr = PetscViewerBinaryGetUseMPIIO(viewer,&useMPIIO);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MPIIO)
if (useMPIIO) {
MPI_File mfdes;
MPI_Offset off;
PetscMPIInt lsize;
PetscInt rstart;
const PetscInt *iarray;
ierr = PetscMPIIntCast(n,&lsize);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIODescriptor(viewer,&mfdes);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIOOffset(viewer,&off);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(is->map,&rstart,NULL);CHKERRQ(ierr);
off += rstart*(MPI_Offset)sizeof(PetscInt); /* off is MPI_Offset, not PetscMPIInt */
ierr = MPI_File_set_view(mfdes,off,MPIU_INT,MPIU_INT,(char*)"native",MPI_INFO_NULL);CHKERRQ(ierr);
ierr = ISGetIndices(is,&iarray);CHKERRQ(ierr);
ierr = MPIU_File_write_all(mfdes,(void*)iarray,lsize,MPIU_INT,MPI_STATUS_IGNORE);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&iarray);CHKERRQ(ierr);
ierr = PetscViewerBinaryAddMPIIOOffset(viewer,N*(MPI_Offset)sizeof(PetscInt));CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#endif
ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr);
/* determine maximum message to arrive */
ierr = MPI_Reduce(&n,&len,1,MPIU_INT,MPI_MAX,0,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlStart(viewer,&message_count,&flowcontrolcount);CHKERRQ(ierr);
if (!rank) {
ierr = PetscBinaryWrite(fdes,isa->idx,n,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscMalloc1(len,&values);CHKERRQ(ierr);
ierr = PetscMPIIntCast(len,&mesgsize);CHKERRQ(ierr);
/* receive and save messages */
for (j=1; j<size; j++) {
ierr = PetscViewerFlowControlStepMaster(viewer,j,&message_count,flowcontrolcount);CHKERRQ(ierr);
ierr = MPI_Recv(values,mesgsize,MPIU_INT,j,tag,PetscObjectComm((PetscObject)is),&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_INT,&mesglen);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fdes,values,(PetscInt)mesglen,PETSC_INT,PETSC_TRUE);CHKERRQ(ierr);
}
ierr = PetscViewerFlowControlEndMaster(viewer,&message_count);CHKERRQ(ierr);
ierr = PetscFree(values);CHKERRQ(ierr);
} else {
ierr = PetscViewerFlowControlStepWorker(viewer,rank,&message_count);CHKERRQ(ierr);
ierr = PetscMPIIntCast(n,&mesgsize);CHKERRQ(ierr);
ierr = MPI_Send(isa->idx,mesgsize,MPIU_INT,0,tag,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlEndWorker(viewer,&message_count);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}