PetscErrorCode OpSolution(Op op,DM dm,Vec U) {
PetscErrorCode ierr;
Vec X;
const PetscScalar *x;
PetscScalar *u;
PetscReal L[3];
PetscInt i,m,bs;
PetscFunctionBegin;
ierr = PetscLogEventBegin(OP_Solution,dm,U,0,0);CHKERRQ(ierr);
ierr = DMGetCoordinates(dm,&X);CHKERRQ(ierr);
ierr = VecStrideMax(X,0,NULL,&L[0]);CHKERRQ(ierr);
ierr = VecStrideMax(X,1,NULL,&L[1]);CHKERRQ(ierr);
ierr = VecStrideMax(X,2,NULL,&L[2]);CHKERRQ(ierr);
ierr = VecGetLocalSize(U,&m);CHKERRQ(ierr);
ierr = VecGetBlockSize(U,&bs);CHKERRQ(ierr);
ierr = VecGetArrayRead(X,&x);CHKERRQ(ierr);
ierr = VecGetArray(U,&u);CHKERRQ(ierr);
for (i=0; i<m/bs; i++) {
ierr = (op->PointwiseSolution)(op,&x[i*3],L,&u[i*bs]);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(X,&x);CHKERRQ(ierr);
ierr = VecRestoreArray(U,&u);CHKERRQ(ierr);
ierr = PetscLogEventEnd(OP_Solution,dm,U,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode BVCreate_Mat(BV bv)
{
PetscErrorCode ierr;
BV_MAT *ctx;
PetscInt nloc,bs;
PetscBool seq;
char str[50];
PetscFunctionBegin;
ierr = PetscNewLog(bv,&ctx);CHKERRQ(ierr);
bv->data = (void*)ctx;
ierr = PetscObjectTypeCompare((PetscObject)bv->t,VECMPI,&ctx->mpi);CHKERRQ(ierr);
if (!ctx->mpi) {
ierr = PetscObjectTypeCompare((PetscObject)bv->t,VECSEQ,&seq);CHKERRQ(ierr);
if (!seq) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot create a BVMAT from a non-standard template vector");
}
ierr = VecGetLocalSize(bv->t,&nloc);CHKERRQ(ierr);
ierr = VecGetBlockSize(bv->t,&bs);CHKERRQ(ierr);
ierr = MatCreateDense(PetscObjectComm((PetscObject)bv->t),nloc,PETSC_DECIDE,PETSC_DECIDE,bv->m,NULL,&ctx->A);CHKERRQ(ierr);
ierr = MatAssemblyBegin(ctx->A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(ctx->A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)bv,(PetscObject)ctx->A);CHKERRQ(ierr);
if (((PetscObject)bv)->name) {
ierr = PetscSNPrintf(str,50,"%s_0",((PetscObject)bv)->name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)ctx->A,str);CHKERRQ(ierr);
}
if (ctx->mpi) {
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)bv->t),bs,nloc,PETSC_DECIDE,NULL,&bv->cv[0]);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)bv->t),bs,nloc,PETSC_DECIDE,NULL,&bv->cv[1]);CHKERRQ(ierr);
} else {
ierr = VecCreateSeqWithArray(PetscObjectComm((PetscObject)bv->t),bs,nloc,NULL,&bv->cv[0]);CHKERRQ(ierr);
ierr = VecCreateSeqWithArray(PetscObjectComm((PetscObject)bv->t),bs,nloc,NULL,&bv->cv[1]);CHKERRQ(ierr);
}
bv->ops->mult = BVMult_Mat;
bv->ops->multvec = BVMultVec_Mat;
bv->ops->multinplace = BVMultInPlace_Mat;
bv->ops->multinplacetrans = BVMultInPlaceTranspose_Mat;
bv->ops->axpy = BVAXPY_Mat;
bv->ops->dot = BVDot_Mat;
bv->ops->dotvec = BVDotVec_Mat;
bv->ops->scale = BVScale_Mat;
bv->ops->norm = BVNorm_Mat;
/*bv->ops->orthogonalize = BVOrthogonalize_Mat;*/
bv->ops->matmult = BVMatMult_Mat;
bv->ops->copy = BVCopy_Mat;
bv->ops->resize = BVResize_Mat;
bv->ops->getcolumn = BVGetColumn_Mat;
bv->ops->restorecolumn = BVRestoreColumn_Mat;
bv->ops->getarray = BVGetArray_Mat;
bv->ops->restorearray = BVRestoreArray_Mat;
bv->ops->destroy = BVDestroy_Mat;
if (!ctx->mpi) bv->ops->view = BVView_Mat;
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode BVCreate_Contiguous(BV bv)
{
PetscErrorCode ierr;
BV_CONTIGUOUS *ctx;
PetscInt j,nloc,bs;
PetscBool seq;
char str[50];
PetscFunctionBegin;
ierr = PetscNewLog(bv,&ctx);CHKERRQ(ierr);
bv->data = (void*)ctx;
ierr = PetscObjectTypeCompare((PetscObject)bv->t,VECMPI,&ctx->mpi);CHKERRQ(ierr);
if (!ctx->mpi) {
ierr = PetscObjectTypeCompare((PetscObject)bv->t,VECSEQ,&seq);CHKERRQ(ierr);
if (!seq) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Cannot create a contiguous BV from a non-standard template vector");
}
ierr = VecGetLocalSize(bv->t,&nloc);CHKERRQ(ierr);
ierr = VecGetBlockSize(bv->t,&bs);CHKERRQ(ierr);
ierr = PetscMalloc1(bv->m*nloc,&ctx->array);CHKERRQ(ierr);
ierr = PetscMemzero(ctx->array,bv->m*nloc*sizeof(PetscScalar));CHKERRQ(ierr);
ierr = PetscMalloc1(bv->m,&ctx->V);CHKERRQ(ierr);
for (j=0;j<bv->m;j++) {
if (ctx->mpi) {
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)bv->t),bs,nloc,PETSC_DECIDE,ctx->array+j*nloc,ctx->V+j);CHKERRQ(ierr);
} else {
ierr = VecCreateSeqWithArray(PetscObjectComm((PetscObject)bv->t),bs,nloc,ctx->array+j*nloc,ctx->V+j);CHKERRQ(ierr);
}
}
ierr = PetscLogObjectParents(bv,bv->m,ctx->V);CHKERRQ(ierr);
if (((PetscObject)bv)->name) {
for (j=0;j<bv->m;j++) {
ierr = PetscSNPrintf(str,50,"%s_%D",((PetscObject)bv)->name,j);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)ctx->V[j],str);CHKERRQ(ierr);
}
}
bv->ops->mult = BVMult_Contiguous;
bv->ops->multvec = BVMultVec_Contiguous;
bv->ops->multinplace = BVMultInPlace_Contiguous;
bv->ops->multinplacetrans = BVMultInPlaceTranspose_Contiguous;
bv->ops->axpy = BVAXPY_Contiguous;
bv->ops->dot = BVDot_Contiguous;
bv->ops->dotvec = BVDotVec_Contiguous;
bv->ops->scale = BVScale_Contiguous;
bv->ops->norm = BVNorm_Contiguous;
/*bv->ops->orthogonalize = BVOrthogonalize_Contiguous;*/
bv->ops->matmult = BVMatMult_Contiguous;
bv->ops->copy = BVCopy_Contiguous;
bv->ops->resize = BVResize_Contiguous;
bv->ops->getcolumn = BVGetColumn_Contiguous;
bv->ops->getarray = BVGetArray_Contiguous;
bv->ops->destroy = BVDestroy_Contiguous;
PetscFunctionReturn(0);
}
PetscErrorCode CreateMesh(MPI_Comm comm, AppCtx *user, DM *dm)
{
PetscInt dim = user->dim;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DMPlexCreateBoxMesh(comm, dim, user->simplex, user->cells, NULL, NULL, NULL, PETSC_TRUE, dm);CHKERRQ(ierr);
{
Parameter *param;
Vec coordinates;
PetscScalar *coords;
PetscReal alpha;
PetscInt cdim, N, bs, i;
ierr = DMGetCoordinateDim(*dm, &cdim);CHKERRQ(ierr);
ierr = DMGetCoordinates(*dm, &coordinates);CHKERRQ(ierr);
ierr = VecGetLocalSize(coordinates, &N);CHKERRQ(ierr);
ierr = VecGetBlockSize(coordinates, &bs);CHKERRQ(ierr);
if (bs != cdim) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Invalid coordinate blocksize %D != embedding dimension %D", bs, cdim);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
ierr = PetscBagGetData(user->bag, (void **) ¶m);CHKERRQ(ierr);
alpha = param->alpha;
for (i = 0; i < N; i += cdim) {
PetscScalar x = coords[i+0];
PetscScalar y = coords[i+1];
coords[i+0] = PetscCosReal(alpha)*x - PetscSinReal(alpha)*y;
coords[i+1] = PetscSinReal(alpha)*x + PetscCosReal(alpha)*y;
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinates(*dm, coordinates);CHKERRQ(ierr);
}
{
DM pdm = NULL;
PetscPartitioner part;
ierr = DMPlexGetPartitioner(*dm, &part);CHKERRQ(ierr);
ierr = PetscPartitionerSetFromOptions(part);CHKERRQ(ierr);
ierr = DMPlexDistribute(*dm, 0, NULL, &pdm);CHKERRQ(ierr);
if (pdm) {
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = pdm;
}
}
ierr = DMSetFromOptions(*dm);CHKERRQ(ierr);
ierr = DMViewFromOptions(*dm, NULL, "-dm_view");CHKERRQ(ierr);
PetscFunctionReturn(0);
}
EXTERN_C_BEGIN
#undef __FUNCT__
#define __FUNCT__ "VecView_DMComposite"
PetscErrorCode VecView_DMComposite(Vec gvec,PetscViewer viewer)
{
DM dm;
PetscErrorCode ierr;
struct DMCompositeLink *next;
PetscBool isdraw;
DM_Composite *com;
PetscFunctionBegin;
ierr = VecGetDM(gvec, &dm);CHKERRQ(ierr);
if (!dm) SETERRQ(((PetscObject)gvec)->comm,PETSC_ERR_ARG_WRONG,"Vector not generated from a DMComposite");
com = (DM_Composite*)dm->data;
next = com->next;
ierr = PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw);CHKERRQ(ierr);
if (!isdraw) {
/* do I really want to call this? */
ierr = VecView_MPI(gvec,viewer);CHKERRQ(ierr);
} else {
PetscInt cnt = 0;
/* loop over packed objects, handling one at at time */
while (next) {
Vec vec;
PetscScalar *array;
PetscInt bs;
/* Should use VecGetSubVector() eventually, but would need to forward the DM for that to work */
ierr = DMGetGlobalVector(next->dm,&vec);CHKERRQ(ierr);
ierr = VecGetArray(gvec,&array);CHKERRQ(ierr);
ierr = VecPlaceArray(vec,array+next->rstart);CHKERRQ(ierr);
ierr = VecRestoreArray(gvec,&array);CHKERRQ(ierr);
ierr = VecView(vec,viewer);CHKERRQ(ierr);
ierr = VecGetBlockSize(vec,&bs);CHKERRQ(ierr);
ierr = VecResetArray(vec);CHKERRQ(ierr);
ierr = DMRestoreGlobalVector(next->dm,&vec);CHKERRQ(ierr);
ierr = PetscViewerDrawBaseAdd(viewer,bs);CHKERRQ(ierr);
cnt += bs;
next = next->next;
}
ierr = PetscViewerDrawBaseAdd(viewer,-cnt);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
dErr dFSSubElementMeshView(dFS fs,dViewer view)
{
dErr err;
dInt nelem,nverts,nconn,*suboff,*subind,n,bs;
dEntTopology *subtopo;
Vec X,Xc;
const dReal *x;
const char *name;
dFunctionBegin;
err = dFSGetSubElementMeshSize(fs,&nelem,&nverts,&nconn);
dCHK(err);
dASSERT(nconn == 8*nelem); /* Assume Hex */
err = dMallocA3(nelem,&subtopo,nelem+1,&suboff,nconn,&subind);
dCHK(err);
err = dFSGetSubElementMesh(fs,nelem,nconn,subtopo,suboff,subind);
dCHK(err);
err = PetscObjectGetName((dObject)fs,&name);
dCHK(err);
err = PetscViewerASCIIPrintf(view,"SubElementMesh name=%s nelem=%D nverts=%D\n",name,nelem,nverts);
dCHK(err);
err = dIntTableView(nelem,8,subind,view,"subconn");
dCHK(err);
err = dFree3(subtopo,suboff,subind);
dCHK(err);
err = dFSGetNodalCoordinatesGlobal(fs,&X);
dCHK(err);
err = VecDohpGetClosure(X,&Xc);
dCHK(err);
err = VecGetLocalSize(Xc,&n);
dCHK(err);
err = VecGetBlockSize(Xc,&bs);
dCHK(err);
err = VecGetArrayRead(Xc,&x);
dCHK(err);
err = dRealTableView(n/bs,bs,x,view,"coords");
dCHK(err);
err = VecRestoreArrayRead(Xc,&x);
dCHK(err);
err = VecDohpRestoreClosure(X,&Xc);
dCHK(err);
dFunctionReturn(0);
}
static PetscErrorCode VecSetUp_NestIS_Private(Vec V,PetscInt nb,IS is[])
{
Vec_Nest *ctx = (Vec_Nest*)V->data;
PetscInt i,offset,m,n,M,N;
PetscErrorCode ierr;
PetscFunctionBegin;
if (is) { /* Do some consistency checks and reference the is */
offset = V->map->rstart;
for (i=0; i<ctx->nb; i++) {
ierr = ISGetSize(is[i],&M);CHKERRQ(ierr);
ierr = VecGetSize(ctx->v[i],&N);CHKERRQ(ierr);
if (M != N) SETERRQ3(PetscObjectComm((PetscObject)V),PETSC_ERR_ARG_INCOMP,"In slot %D, IS of size %D is not compatible with Vec of size %D",i,M,N);
ierr = ISGetLocalSize(is[i],&m);CHKERRQ(ierr);
ierr = VecGetLocalSize(ctx->v[i],&n);CHKERRQ(ierr);
if (m != n) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"In slot %D, IS of local size %D is not compatible with Vec of local size %D",i,m,n);
#if defined(PETSC_USE_DEBUG)
{ /* This test can be expensive */
PetscInt start;
PetscBool contiguous;
ierr = ISContiguousLocal(is[i],offset,offset+n,&start,&contiguous);CHKERRQ(ierr);
if (!contiguous) SETERRQ1(PetscObjectComm((PetscObject)V),PETSC_ERR_SUP,"Index set %D is not contiguous with layout of matching vector",i);
if (start != 0) SETERRQ1(PetscObjectComm((PetscObject)V),PETSC_ERR_SUP,"Index set %D introduces overlap or a hole",i);
}
#endif
ierr = PetscObjectReference((PetscObject)is[i]);CHKERRQ(ierr);
ctx->is[i] = is[i];
offset += n;
}
} else { /* Create a contiguous ISStride for each entry */
offset = V->map->rstart;
for (i=0; i<ctx->nb; i++) {
PetscInt bs;
ierr = VecGetLocalSize(ctx->v[i],&n);CHKERRQ(ierr);
ierr = VecGetBlockSize(ctx->v[i],&bs);CHKERRQ(ierr);
ierr = ISCreateStride(((PetscObject)ctx->v[i])->comm,n,offset,1,&ctx->is[i]);CHKERRQ(ierr);
ierr = ISSetBlockSize(ctx->is[i],bs);CHKERRQ(ierr);
offset += n;
}
}
PetscFunctionReturn(0);
}
void LData::resetData(Vec vec, const std::vector<int>& nonlocal_petsc_indices, const bool manage_petsc_vec)
{
restoreArrays();
int ierr;
if (d_managing_petsc_vec)
{
ierr = VecDestroy(&d_global_vec);
IBTK_CHKERRQ(ierr);
}
// Take ownership of new Vec
d_global_vec = vec;
d_managing_petsc_vec = manage_petsc_vec;
int depth;
ierr = VecGetBlockSize(d_global_vec, &depth);
IBTK_CHKERRQ(ierr);
#if !defined(NDEBUG)
TBOX_ASSERT(depth >= 0);
#endif
d_depth = depth;
int global_node_count;
ierr = VecGetSize(d_global_vec, &global_node_count);
IBTK_CHKERRQ(ierr);
#if !defined(NDEBUG)
TBOX_ASSERT(global_node_count >= 0);
#endif
d_global_node_count = global_node_count;
d_global_node_count /= d_depth;
int local_node_count;
ierr = VecGetLocalSize(d_global_vec, &local_node_count);
IBTK_CHKERRQ(ierr);
#if !defined(NDEBUG)
TBOX_ASSERT(local_node_count >= 0);
#endif
d_local_node_count = local_node_count;
d_local_node_count /= d_depth;
d_nonlocal_petsc_indices = nonlocal_petsc_indices;
d_ghost_node_count = static_cast<int>(d_nonlocal_petsc_indices.size());
return;
} // resetData
static PetscErrorCode ProjectToUnitSphere(DM dm)
{
Vec coordinates;
PetscScalar *coords;
PetscInt Nv, v, dim, d;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = VecGetLocalSize(coordinates, &Nv);CHKERRQ(ierr);
ierr = VecGetBlockSize(coordinates, &dim);CHKERRQ(ierr);
Nv /= dim;
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
for (v = 0; v < Nv; ++v) {
PetscReal r = 0.0;
for (d = 0; d < dim; ++d) r += PetscSqr(PetscRealPart(coords[v*dim+d]));
r = PetscSqrtReal(r);
for (d = 0; d < dim; ++d) coords[v*dim+d] /= r;
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
LData::LData(const std::string& name,
Vec vec,
const std::vector<int>& nonlocal_petsc_indices,
const bool manage_petsc_vec)
: d_name(name), d_global_node_count(0), d_local_node_count(0), d_ghost_node_count(0), d_depth(0),
d_nonlocal_petsc_indices(nonlocal_petsc_indices), d_global_vec(vec), d_managing_petsc_vec(manage_petsc_vec),
d_array(NULL), d_boost_array(NULL), d_boost_local_array(NULL), d_boost_vec_array(NULL),
d_boost_local_vec_array(NULL), d_ghosted_local_vec(NULL), d_ghosted_local_array(NULL),
d_boost_ghosted_local_array(NULL), d_boost_vec_ghosted_local_array(NULL)
{
int ierr;
int depth;
ierr = VecGetBlockSize(d_global_vec, &depth);
IBTK_CHKERRQ(ierr);
#if !defined(NDEBUG)
TBOX_ASSERT(depth >= 0);
#endif
d_depth = depth;
int global_node_count;
ierr = VecGetSize(d_global_vec, &global_node_count);
IBTK_CHKERRQ(ierr);
#if !defined(NDEBUG)
TBOX_ASSERT(global_node_count >= 0);
#endif
d_global_node_count = global_node_count;
d_global_node_count /= d_depth;
int local_node_count;
ierr = VecGetLocalSize(d_global_vec, &local_node_count);
IBTK_CHKERRQ(ierr);
#if !defined(NDEBUG)
TBOX_ASSERT(local_node_count >= 0);
#endif
d_local_node_count = local_node_count;
d_local_node_count /= d_depth;
d_ghost_node_count = static_cast<int>(d_nonlocal_petsc_indices.size());
return;
} // LData
PetscErrorCode BVResize_Contiguous(BV bv,PetscInt m,PetscBool copy)
{
PetscErrorCode ierr;
BV_CONTIGUOUS *ctx = (BV_CONTIGUOUS*)bv->data;
PetscInt j,bs;
PetscScalar *newarray;
Vec *newV;
char str[50];
PetscFunctionBegin;
ierr = VecGetBlockSize(bv->t,&bs);CHKERRQ(ierr);
ierr = PetscMalloc1(m*bv->n,&newarray);CHKERRQ(ierr);
ierr = PetscMemzero(newarray,m*bv->n*sizeof(PetscScalar));CHKERRQ(ierr);
ierr = PetscMalloc1(m,&newV);CHKERRQ(ierr);
for (j=0;j<m;j++) {
if (ctx->mpi) {
ierr = VecCreateMPIWithArray(PetscObjectComm((PetscObject)bv->t),bs,bv->n,PETSC_DECIDE,newarray+j*bv->n,newV+j);CHKERRQ(ierr);
} else {
ierr = VecCreateSeqWithArray(PetscObjectComm((PetscObject)bv->t),bs,bv->n,newarray+j*bv->n,newV+j);CHKERRQ(ierr);
}
}
ierr = PetscLogObjectParents(bv,m,newV);CHKERRQ(ierr);
if (((PetscObject)bv)->name) {
for (j=0;j<m;j++) {
ierr = PetscSNPrintf(str,50,"%s_%D",((PetscObject)bv)->name,j);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)newV[j],str);CHKERRQ(ierr);
}
}
if (copy) {
ierr = PetscMemcpy(newarray,ctx->array,PetscMin(m,bv->m)*bv->n*sizeof(PetscScalar));CHKERRQ(ierr);
}
ierr = VecDestroyVecs(bv->m,&ctx->V);CHKERRQ(ierr);
ctx->V = newV;
ierr = PetscFree(ctx->array);CHKERRQ(ierr);
ctx->array = newarray;
PetscFunctionReturn(0);
}
PetscErrorCode VecSetValuesBlocked_Seq(Vec xin,PetscInt ni,const PetscInt ix[],const PetscScalar yin[],InsertMode m)
{
PetscScalar *xx,*y = (PetscScalar*)yin;
PetscInt i,bs,start,j;
PetscErrorCode ierr;
/*
For optimization could treat bs = 2, 3, 4, 5 as special cases with loop unrolling
*/
PetscFunctionBegin;
ierr = VecGetBlockSize(xin,&bs);CHKERRQ(ierr);
ierr = VecGetArray(xin,&xx);CHKERRQ(ierr);
if (m == INSERT_VALUES) {
for (i=0; i<ni; i++) {
start = bs*ix[i];
if (start < 0) continue;
#if defined(PETSC_USE_DEBUG)
if (start >= xin->map->n) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Out of range index value %D maximum %D",start,xin->map->n);
#endif
for (j=0; j<bs; j++) xx[start+j] = y[j];
y += bs;
}
} else {
for (i=0; i<ni; i++) {
start = bs*ix[i];
if (start < 0) continue;
#if defined(PETSC_USE_DEBUG)
if (start >= xin->map->n) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Out of range index value %D maximum %D",start,xin->map->n);
#endif
for (j=0; j<bs; j++) xx[start+j] += y[j];
y += bs;
}
}
ierr = VecRestoreArray(xin,&xx);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
This should handle properly the cases where PetscInt is 32 or 64 and hsize_t is 32 or 64. These means properly casting with
checks back and forth between the two types of variables.
*/
PetscErrorCode VecLoad_HDF5(Vec xin, PetscViewer viewer)
{
hid_t file_id, group, dset_id, filespace, memspace, plist_id;
hsize_t rdim, dim;
hsize_t dims[4], count[4], offset[4];
herr_t status;
PetscInt n, N, bs = 1, bsInd, lenInd, low, timestep;
PetscScalar *x;
const char *vecname;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscViewerHDF5OpenGroup(viewer, &file_id, &group);CHKERRQ(ierr);
ierr = PetscViewerHDF5GetTimestep(viewer, ×tep);CHKERRQ(ierr);
ierr = VecGetBlockSize(xin,&bs);CHKERRQ(ierr);
/* Create the dataset with default properties and close filespace */
ierr = PetscObjectGetName((PetscObject)xin,&vecname);CHKERRQ(ierr);
#if (H5_VERS_MAJOR * 10000 + H5_VERS_MINOR * 100 + H5_VERS_RELEASE >= 10800)
dset_id = H5Dopen2(group, vecname, H5P_DEFAULT);
#else
dset_id = H5Dopen(group, vecname);
#endif
if (dset_id == -1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Dopen() with Vec named %s",vecname);
/* Retrieve the dataspace for the dataset */
filespace = H5Dget_space(dset_id);
if (filespace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Dget_space()");
dim = 0;
if (timestep >= 0) ++dim;
++dim;
if (bs >= 1) ++dim;
#if defined(PETSC_USE_COMPLEX)
++dim;
#endif
rdim = H5Sget_simple_extent_dims(filespace, dims, NULL);
#if defined(PETSC_USE_COMPLEX)
bsInd = rdim-2;
#else
bsInd = rdim-1;
#endif
lenInd = timestep >= 0 ? 1 : 0;
if (rdim != dim) {
if (rdim == dim+1 && bs == -1) bs = dims[bsInd];
else SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Dimension of array in file %d not %d as expected",rdim,dim);
} else if (bs >= 1 && bs != (PetscInt) dims[bsInd]) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_FILE_UNEXPECTED, "Block size %d specified for vector does not match blocksize in file %d",bs,dims[bsInd]);
/* Set Vec sizes,blocksize,and type if not already set */
if ((xin)->map->n < 0 && (xin)->map->N < 0) {
ierr = VecSetSizes(xin, PETSC_DECIDE, dims[lenInd]*bs);CHKERRQ(ierr);
}
/* If sizes and type already set,check if the vector global size is correct */
ierr = VecGetSize(xin, &N);CHKERRQ(ierr);
if (N/bs != (PetscInt) dims[lenInd]) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_FILE_UNEXPECTED, "Vector in file different length (%d) then input vector (%d)", (PetscInt) dims[lenInd], N/bs);
/* Each process defines a dataset and reads it from the hyperslab in the file */
ierr = VecGetLocalSize(xin, &n);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
count[dim] = 1;
++dim;
}
ierr = PetscHDF5IntCast(n/bs,count + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
count[dim] = bs;
++dim;
}
#if defined(PETSC_USE_COMPLEX)
count[dim] = 2;
++dim;
#endif
memspace = H5Screate_simple(dim, count, NULL);
if (memspace == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Screate_simple()");
/* Select hyperslab in the file */
ierr = VecGetOwnershipRange(xin, &low, NULL);CHKERRQ(ierr);
dim = 0;
if (timestep >= 0) {
offset[dim] = timestep;
++dim;
}
ierr = PetscHDF5IntCast(low/bs,offset + dim);CHKERRQ(ierr);
++dim;
if (bs >= 1) {
offset[dim] = 0;
++dim;
}
#if defined(PETSC_USE_COMPLEX)
offset[dim] = 0;
++dim;
#endif
status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, count, NULL);CHKERRQ(status);
/* Create property list for collective dataset read */
plist_id = H5Pcreate(H5P_DATASET_XFER);
if (plist_id == -1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Could not H5Pcreate()");
#if defined(PETSC_HAVE_H5PSET_FAPL_MPIO)
status = H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE);CHKERRQ(status);
#endif
/* To write dataset independently use H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT) */
ierr = VecGetArray(xin, &x);CHKERRQ(ierr);
status = H5Dread(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, x);CHKERRQ(status);
ierr = VecRestoreArray(xin, &x);CHKERRQ(ierr);
/* Close/release resources */
if (group != file_id) {
status = H5Gclose(group);CHKERRQ(status);
}
status = H5Pclose(plist_id);CHKERRQ(status);
status = H5Sclose(filespace);CHKERRQ(status);
status = H5Sclose(memspace);CHKERRQ(status);
status = H5Dclose(dset_id);CHKERRQ(status);
ierr = VecAssemblyBegin(xin);CHKERRQ(ierr);
ierr = VecAssemblyEnd(xin);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
MPI_Comm comm;
DM dm;
Vec v, nv, rv, coord;
PetscBool test_read = PETSC_FALSE, verbose = PETSC_FALSE, flg;
PetscViewer hdf5Viewer;
PetscInt dim = 2;
PetscInt numFields = 1;
PetscInt numBC = 0;
PetscInt numComp[1] = {2};
PetscInt numDof[3] = {2, 0, 0};
PetscInt bcFields[1] = {0};
IS bcPoints[1] = {NULL};
PetscSection section;
PetscReal norm;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, (char *) 0, help);CHKERRQ(ierr);
comm = PETSC_COMM_WORLD;
ierr = PetscOptionsBegin(PETSC_COMM_WORLD,"","Test Options","none");CHKERRQ(ierr);
ierr = PetscOptionsBool("-test_read","Test reading from the HDF5 file","",PETSC_FALSE,&test_read,NULL);CHKERRQ(ierr);
ierr = PetscOptionsBool("-verbose","print the Vecs","",PETSC_FALSE,&verbose,NULL);CHKERRQ(ierr);
ierr = PetscOptionsInt("-dim","the dimension of the problem","",2,&dim,NULL);CHKERRQ(ierr);
ierr = PetscOptionsEnd();
ierr = DMPlexCreateBoxMesh(comm, dim, PETSC_TRUE, &dm);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
numDof[0] = dim;
ierr = DMPlexCreateSection(dm, dim, numFields, numComp, numDof, numBC, bcFields, bcPoints, NULL, NULL, §ion);CHKERRQ(ierr);
ierr = DMSetDefaultSection(dm, section);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ion);CHKERRQ(ierr);
ierr = DMSetUseNatural(dm, PETSC_TRUE);CHKERRQ(ierr);
{
DM dmDist;
ierr = DMPlexDistribute(dm, 0, NULL, &dmDist);CHKERRQ(ierr);
if (dmDist) {
ierr = DMDestroy(&dm);CHKERRQ(ierr);
dm = dmDist;
}
}
ierr = DMCreateGlobalVector(dm, &v);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) v, "V");CHKERRQ(ierr);
ierr = DMGetCoordinates(dm, &coord);CHKERRQ(ierr);
ierr = VecCopy(coord, v);CHKERRQ(ierr);
if (verbose) {
PetscInt size, bs;
ierr = VecGetSize(v, &size);CHKERRQ(ierr);
ierr = VecGetBlockSize(v, &bs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "==== original V in global ordering. size==%d\tblock size=%d\n", size, bs);CHKERRQ(ierr);
ierr = VecView(v, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = DMCreateGlobalVector(dm, &nv);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) nv, "NV");CHKERRQ(ierr);
ierr = DMPlexGlobalToNaturalBegin(dm, v, nv);CHKERRQ(ierr);
ierr = DMPlexGlobalToNaturalEnd(dm, v, nv);CHKERRQ(ierr);
if (verbose) {
PetscInt size, bs;
ierr = VecGetSize(nv, &size);CHKERRQ(ierr);
ierr = VecGetBlockSize(nv, &bs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "==== V in natural ordering. size==%d\tblock size=%d\n", size, bs);CHKERRQ(ierr);
ierr = VecView(nv, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecViewFromOptions(v, NULL, "-global_vec_view");CHKERRQ(ierr);
if (test_read) {
ierr = DMCreateGlobalVector(dm, &rv);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) rv, "V");CHKERRQ(ierr);
/* Test native read */
ierr = PetscViewerHDF5Open(comm, "V.h5", FILE_MODE_READ, &hdf5Viewer);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(hdf5Viewer, PETSC_VIEWER_NATIVE);CHKERRQ(ierr);
ierr = VecLoad(rv, hdf5Viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&hdf5Viewer);CHKERRQ(ierr);
if (verbose) {
PetscInt size, bs;
ierr = VecGetSize(rv, &size);CHKERRQ(ierr);
ierr = VecGetBlockSize(rv, &bs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "==== Vector from file. size==%d\tblock size=%d\n", size, bs);CHKERRQ(ierr);
ierr = VecView(rv, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecEqual(rv, v, &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "V and RV are equal\n");CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD, "V and RV are not equal\n\n");CHKERRQ(ierr);
ierr = VecAXPY(rv, -1.0, v);CHKERRQ(ierr);
ierr = VecNorm(rv, NORM_INFINITY, &norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "diff norm is = %g\n", (double) norm);CHKERRQ(ierr);
ierr = VecView(rv, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Test raw read */
ierr = PetscViewerHDF5Open(comm, "V.h5", FILE_MODE_READ, &hdf5Viewer);CHKERRQ(ierr);
ierr = VecLoad(rv, hdf5Viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&hdf5Viewer);CHKERRQ(ierr);
if (verbose) {
PetscInt size, bs;
ierr = VecGetSize(rv, &size);CHKERRQ(ierr);
ierr = VecGetBlockSize(rv, &bs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "==== Vector from file. size==%d\tblock size=%d\n", size, bs);CHKERRQ(ierr);
ierr = VecView(rv, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecEqual(rv, nv, &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD, "NV and RV are equal\n");CHKERRQ(ierr);
} else {
ierr = PetscPrintf(PETSC_COMM_WORLD, "NV and RV are not equal\n\n");CHKERRQ(ierr);
ierr = VecAXPY(rv, -1.0, v);CHKERRQ(ierr);
ierr = VecNorm(rv, NORM_INFINITY, &norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "diff norm is = %g\n", (double) norm);CHKERRQ(ierr);
ierr = VecView(rv, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecDestroy(&rv);CHKERRQ(ierr);
}
ierr = VecDestroy(&nv);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode VecNestSetSubVec_Private(Vec X,PetscInt idxm,Vec x)
{
Vec_Nest *bx = (Vec_Nest*)X->data;
PetscInt i,offset=0,n=0,bs;
IS is;
PetscErrorCode ierr;
PetscBool issame = PETSC_FALSE;
PetscInt N=0;
/* check if idxm < bx->nb */
if (idxm >= bx->nb) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Out of range index value %D maximum %D",idxm,bx->nb);
PetscFunctionBegin;
ierr = VecDestroy(&bx->v[idxm]);CHKERRQ(ierr); /* destroy the existing vector */
ierr = VecDuplicate(x,&bx->v[idxm]);CHKERRQ(ierr); /* duplicate the layout of given vector */
ierr = VecCopy(x,bx->v[idxm]);CHKERRQ(ierr); /* copy the contents of the given vector */
/* check if we need to update the IS for the block */
offset = X->map->rstart;
for (i=0; i<idxm; i++) {
n=0;
ierr = VecGetLocalSize(bx->v[i],&n);CHKERRQ(ierr);
offset += n;
}
/* get the local size and block size */
ierr = VecGetLocalSize(x,&n);CHKERRQ(ierr);
ierr = VecGetBlockSize(x,&bs);CHKERRQ(ierr);
/* create the new IS */
ierr = ISCreateStride(PetscObjectComm((PetscObject)x),n,offset,1,&is);CHKERRQ(ierr);
ierr = ISSetBlockSize(is,bs);CHKERRQ(ierr);
/* check if they are equal */
ierr = ISEqual(is,bx->is[idxm],&issame);CHKERRQ(ierr);
if (!issame) {
/* The IS of given vector has a different layout compared to the existing block vector.
Destroy the existing reference and update the IS. */
ierr = ISDestroy(&bx->is[idxm]);CHKERRQ(ierr);
ierr = ISDuplicate(is,&bx->is[idxm]);CHKERRQ(ierr);
ierr = ISCopy(is,bx->is[idxm]);CHKERRQ(ierr);
offset += n;
/* Since the current IS[idxm] changed, we need to update all the subsequent IS */
for (i=idxm+1; i<bx->nb; i++) {
/* get the local size and block size */
ierr = VecGetLocalSize(bx->v[i],&n);CHKERRQ(ierr);
ierr = VecGetBlockSize(bx->v[i],&bs);CHKERRQ(ierr);
/* destroy the old and create the new IS */
ierr = ISDestroy(&bx->is[i]);CHKERRQ(ierr);
ierr = ISCreateStride(((PetscObject)bx->v[i])->comm,n,offset,1,&bx->is[i]);CHKERRQ(ierr);
ierr = ISSetBlockSize(bx->is[i],bs);CHKERRQ(ierr);
offset += n;
}
n=0;
ierr = VecSize_Nest_Recursive(X,PETSC_TRUE,&N);CHKERRQ(ierr);
ierr = VecSize_Nest_Recursive(X,PETSC_FALSE,&n);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(X->map,N);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(X->map,n);CHKERRQ(ierr);
}
ierr = ISDestroy(&is);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@
MatNullSpaceCreateRigidBody - create rigid body modes from coordinates
Collective on Vec
Input Argument:
. coords - block of coordinates of each node, must have block size set
Output Argument:
. sp - the null space
Level: advanced
Notes:
If you are solving an elasticity problems you should likely use this, in conjunction with ee MatSetNearNullspace(), to provide information that
the PCGAMG preconditioner can use to construct a much more efficient preconditioner.
If you are solving an elasticity problem with pure Neumann boundary conditions you can use this in conjunction with MatSetNullspace() to
provide this information to the linear solver so it can handle the null space appropriately in the linear solution.
.seealso: MatNullSpaceCreate(), MatSetNearNullspace(), MatSetNullspace()
@*/
PetscErrorCode MatNullSpaceCreateRigidBody(Vec coords,MatNullSpace *sp)
{
PetscErrorCode ierr;
const PetscScalar *x;
PetscScalar *v[6],dots[5];
Vec vec[6];
PetscInt n,N,dim,nmodes,i,j;
PetscReal sN;
PetscFunctionBegin;
ierr = VecGetBlockSize(coords,&dim);CHKERRQ(ierr);
ierr = VecGetLocalSize(coords,&n);CHKERRQ(ierr);
ierr = VecGetSize(coords,&N);CHKERRQ(ierr);
n /= dim;
N /= dim;
sN = 1./PetscSqrtReal((PetscReal)N);
switch (dim) {
case 1:
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)coords),PETSC_TRUE,0,NULL,sp);CHKERRQ(ierr);
break;
case 2:
case 3:
nmodes = (dim == 2) ? 3 : 6;
ierr = VecCreate(PetscObjectComm((PetscObject)coords),&vec[0]);CHKERRQ(ierr);
ierr = VecSetSizes(vec[0],dim*n,dim*N);CHKERRQ(ierr);
ierr = VecSetBlockSize(vec[0],dim);CHKERRQ(ierr);
ierr = VecSetUp(vec[0]);CHKERRQ(ierr);
for (i=1; i<nmodes; i++) {ierr = VecDuplicate(vec[0],&vec[i]);CHKERRQ(ierr);}
for (i=0; i<nmodes; i++) {ierr = VecGetArray(vec[i],&v[i]);CHKERRQ(ierr);}
ierr = VecGetArrayRead(coords,&x);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (dim == 2) {
v[0][i*2+0] = sN;
v[0][i*2+1] = 0.;
v[1][i*2+0] = 0.;
v[1][i*2+1] = sN;
/* Rotations */
v[2][i*2+0] = -x[i*2+1];
v[2][i*2+1] = x[i*2+0];
} else {
v[0][i*3+0] = sN;
v[0][i*3+1] = 0.;
v[0][i*3+2] = 0.;
v[1][i*3+0] = 0.;
v[1][i*3+1] = sN;
v[1][i*3+2] = 0.;
v[2][i*3+0] = 0.;
v[2][i*3+1] = 0.;
v[2][i*3+2] = sN;
v[3][i*3+0] = x[i*3+1];
v[3][i*3+1] = -x[i*3+0];
v[3][i*3+2] = 0.;
v[4][i*3+0] = 0.;
v[4][i*3+1] = -x[i*3+2];
v[4][i*3+2] = x[i*3+1];
v[5][i*3+0] = x[i*3+2];
v[5][i*3+1] = 0.;
v[5][i*3+2] = -x[i*3+0];
}
}
for (i=0; i<nmodes; i++) {ierr = VecRestoreArray(vec[i],&v[i]);CHKERRQ(ierr);}
ierr = VecRestoreArrayRead(coords,&x);CHKERRQ(ierr);
for (i=dim; i<nmodes; i++) {
/* Orthonormalize vec[i] against vec[0:i-1] */
ierr = VecMDot(vec[i],i,vec,dots);CHKERRQ(ierr);
for (j=0; j<i; j++) dots[j] *= -1.;
ierr = VecMAXPY(vec[i],i,dots,vec);CHKERRQ(ierr);
ierr = VecNormalize(vec[i],NULL);CHKERRQ(ierr);
}
ierr = MatNullSpaceCreate(PetscObjectComm((PetscObject)coords),PETSC_FALSE,nmodes,vec,sp);CHKERRQ(ierr);
for (i=0; i<nmodes; i++) {ierr = VecDestroy(&vec[i]);CHKERRQ(ierr);}
}
PetscFunctionReturn(0);
}
PetscErrorCode VecView_Seq_ASCII(Vec xin,PetscViewer viewer)
{
PetscErrorCode ierr;
PetscInt i,n = xin->map->n;
const char *name;
PetscViewerFormat format;
const PetscScalar *xv;
PetscFunctionBegin;
ierr = VecGetArrayRead(xin,&xv);CHKERRQ(ierr);
ierr = PetscViewerGetFormat(viewer,&format);CHKERRQ(ierr);
if (format == PETSC_VIEWER_ASCII_MATLAB) {
ierr = PetscObjectGetName((PetscObject)xin,&name);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%s = [\n",name);CHKERRQ(ierr);
for (i=0; i<n; i++) {
#if defined(PETSC_USE_COMPLEX)
if (PetscImaginaryPart(xv[i]) > 0.0) {
ierr = PetscViewerASCIIPrintf(viewer,"%18.16e + %18.16ei\n",(double)PetscRealPart(xv[i]),(double)PetscImaginaryPart(xv[i]));CHKERRQ(ierr);
} else if (PetscImaginaryPart(xv[i]) < 0.0) {
ierr = PetscViewerASCIIPrintf(viewer,"%18.16e - %18.16ei\n",(double)PetscRealPart(xv[i]),-(double)PetscImaginaryPart(xv[i]));CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"%18.16e\n",(double)PetscRealPart(xv[i]));CHKERRQ(ierr);
}
#else
ierr = PetscViewerASCIIPrintf(viewer,"%18.16e\n",(double) xv[i]);CHKERRQ(ierr);
#endif
}
ierr = PetscViewerASCIIPrintf(viewer,"];\n");CHKERRQ(ierr);
} else if (format == PETSC_VIEWER_ASCII_SYMMODU) {
for (i=0; i<n; i++) {
#if defined(PETSC_USE_COMPLEX)
ierr = PetscViewerASCIIPrintf(viewer,"%18.16e %18.16e\n",(double)PetscRealPart(xv[i]),(double)PetscImaginaryPart(xv[i]));CHKERRQ(ierr);
#else
ierr = PetscViewerASCIIPrintf(viewer,"%18.16e\n",(double)xv[i]);CHKERRQ(ierr);
#endif
}
} else if (format == PETSC_VIEWER_ASCII_VTK || format == PETSC_VIEWER_ASCII_VTK_CELL) {
/*
state 0: No header has been output
state 1: Only POINT_DATA has been output
state 2: Only CELL_DATA has been output
state 3: Output both, POINT_DATA last
state 4: Output both, CELL_DATA last
*/
static PetscInt stateId = -1;
int outputState = 0;
PetscBool hasState;
int doOutput = 0;
PetscInt bs, b;
if (stateId < 0) {
ierr = PetscObjectComposedDataRegister(&stateId);CHKERRQ(ierr);
}
ierr = PetscObjectComposedDataGetInt((PetscObject) viewer, stateId, outputState, hasState);CHKERRQ(ierr);
if (!hasState) outputState = 0;
ierr = PetscObjectGetName((PetscObject) xin, &name);CHKERRQ(ierr);
ierr = VecGetBlockSize(xin, &bs);CHKERRQ(ierr);
if ((bs < 1) || (bs > 3)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE, "VTK can only handle 3D objects, but vector dimension is %d", bs);
if (format == PETSC_VIEWER_ASCII_VTK) {
if (outputState == 0) {
outputState = 1;
doOutput = 1;
} else if (outputState == 1) doOutput = 0;
else if (outputState == 2) {
outputState = 3;
doOutput = 1;
} else if (outputState == 3) doOutput = 0;
else if (outputState == 4) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE, "Tried to output POINT_DATA again after intervening CELL_DATA");
if (doOutput) {
ierr = PetscViewerASCIIPrintf(viewer, "POINT_DATA %d\n", n/bs);CHKERRQ(ierr);
}
} else {
if (outputState == 0) {
outputState = 2;
doOutput = 1;
} else if (outputState == 1) {
outputState = 4;
doOutput = 1;
} else if (outputState == 2) doOutput = 0;
else if (outputState == 3) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE, "Tried to output CELL_DATA again after intervening POINT_DATA");
else if (outputState == 4) doOutput = 0;
if (doOutput) {
ierr = PetscViewerASCIIPrintf(viewer, "CELL_DATA %d\n", n);CHKERRQ(ierr);
}
}
ierr = PetscObjectComposedDataSetInt((PetscObject) viewer, stateId, outputState);CHKERRQ(ierr);
if (name) {
if (bs == 3) {
ierr = PetscViewerASCIIPrintf(viewer, "VECTORS %s double\n", name);CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer, "SCALARS %s double %d\n", name, bs);CHKERRQ(ierr);
}
} else {
ierr = PetscViewerASCIIPrintf(viewer, "SCALARS scalars double %d\n", bs);CHKERRQ(ierr);
}
if (bs != 3) {
ierr = PetscViewerASCIIPrintf(viewer, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
}
for (i=0; i<n/bs; i++) {
for (b=0; b<bs; b++) {
if (b > 0) {
ierr = PetscViewerASCIIPrintf(viewer," ");CHKERRQ(ierr);
}
#if !defined(PETSC_USE_COMPLEX)
ierr = PetscViewerASCIIPrintf(viewer,"%g",(double)xv[i*bs+b]);CHKERRQ(ierr);
#endif
}
ierr = PetscViewerASCIIPrintf(viewer,"\n");CHKERRQ(ierr);
}
} else if (format == PETSC_VIEWER_ASCII_VTK_COORDS) {
PetscInt bs, b;
ierr = VecGetBlockSize(xin, &bs);CHKERRQ(ierr);
if ((bs < 1) || (bs > 3)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE, "VTK can only handle 3D objects, but vector dimension is %d", bs);
for (i=0; i<n/bs; i++) {
for (b=0; b<bs; b++) {
if (b > 0) {
ierr = PetscViewerASCIIPrintf(viewer," ");CHKERRQ(ierr);
}
#if !defined(PETSC_USE_COMPLEX)
ierr = PetscViewerASCIIPrintf(viewer,"%g",(double)xv[i*bs+b]);CHKERRQ(ierr);
#endif
}
for (b=bs; b<3; b++) {
ierr = PetscViewerASCIIPrintf(viewer," 0.0");CHKERRQ(ierr);
}
ierr = PetscViewerASCIIPrintf(viewer,"\n");CHKERRQ(ierr);
}
} else if (format == PETSC_VIEWER_ASCII_PCICE) {
PetscInt bs, b;
ierr = VecGetBlockSize(xin, &bs);CHKERRQ(ierr);
if ((bs < 1) || (bs > 3)) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE, "PCICE can only handle up to 3D objects, but vector dimension is %d", bs);
ierr = PetscViewerASCIIPrintf(viewer,"%D\n", xin->map->N/bs);CHKERRQ(ierr);
for (i=0; i<n/bs; i++) {
ierr = PetscViewerASCIIPrintf(viewer,"%7D ", i+1);CHKERRQ(ierr);
for (b=0; b<bs; b++) {
if (b > 0) {
ierr = PetscViewerASCIIPrintf(viewer," ");CHKERRQ(ierr);
}
#if !defined(PETSC_USE_COMPLEX)
ierr = PetscViewerASCIIPrintf(viewer,"% 12.5E",(double)xv[i*bs+b]);CHKERRQ(ierr);
#endif
}
ierr = PetscViewerASCIIPrintf(viewer,"\n");CHKERRQ(ierr);
}
} else if (format == PETSC_VIEWER_ASCII_INFO || format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
/* No info */
} else {
for (i=0; i<n; i++) {
if (format == PETSC_VIEWER_ASCII_INDEX) {
ierr = PetscViewerASCIIPrintf(viewer,"%D: ",i);CHKERRQ(ierr);
}
#if defined(PETSC_USE_COMPLEX)
if (PetscImaginaryPart(xv[i]) > 0.0) {
ierr = PetscViewerASCIIPrintf(viewer,"%g + %g i\n",(double)PetscRealPart(xv[i]),(double)PetscImaginaryPart(xv[i]));CHKERRQ(ierr);
} else if (PetscImaginaryPart(xv[i]) < 0.0) {
ierr = PetscViewerASCIIPrintf(viewer,"%g - %g i\n",(double)PetscRealPart(xv[i]),-(double)PetscImaginaryPart(xv[i]));CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"%g\n",(double)PetscRealPart(xv[i]));CHKERRQ(ierr);
}
#else
ierr = PetscViewerASCIIPrintf(viewer,"%g\n",(double)xv[i]);CHKERRQ(ierr);
#endif
}
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(xin,&xv);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETSC_STDCALL vecgetblocksize_(Vec v,PetscInt *bs, int *__ierr ){
*__ierr = VecGetBlockSize(
(Vec)PetscToPointer((v) ),bs);
}
/*@C
DMPlexDistribute - Distributes the mesh and any associated sections.
Not Collective
Input Parameter:
+ dm - The original DMPlex object
. partitioner - The partitioning package, or NULL for the default
- overlap - The overlap of partitions, 0 is the default
Output Parameter:
+ sf - The PetscSF used for point distribution
- parallelMesh - The distributed DMPlex object, or NULL
Note: If the mesh was not distributed, the return value is NULL.
The user can control the definition of adjacency for the mesh using DMPlexGetAdjacencyUseCone() and
DMPlexSetAdjacencyUseClosure(). They should choose the combination appropriate for the function
representation on the mesh.
Level: intermediate
.keywords: mesh, elements
.seealso: DMPlexCreate(), DMPlexDistributeByFace(), DMPlexSetAdjacencyUseCone(), DMPlexSetAdjacencyUseClosure()
@*/
PetscErrorCode DMPlexDistribute(DM dm, const char partitioner[], PetscInt overlap, PetscSF *sf, DM *dmParallel)
{
DM_Plex *mesh = (DM_Plex*) dm->data, *pmesh;
MPI_Comm comm;
const PetscInt height = 0;
PetscInt dim, numRemoteRanks;
IS origCellPart, origPart, cellPart, part;
PetscSection origCellPartSection, origPartSection, cellPartSection, partSection;
PetscSFNode *remoteRanks;
PetscSF partSF, pointSF, coneSF;
ISLocalToGlobalMapping renumbering;
PetscSection originalConeSection, newConeSection;
PetscInt *remoteOffsets;
PetscInt *cones, *newCones, newConesSize;
PetscBool flg;
PetscMPIInt rank, numProcs, p;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
if (sf) PetscValidPointer(sf,4);
PetscValidPointer(dmParallel,5);
ierr = PetscLogEventBegin(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &numProcs);CHKERRQ(ierr);
*dmParallel = NULL;
if (numProcs == 1) PetscFunctionReturn(0);
ierr = DMPlexGetDimension(dm, &dim);CHKERRQ(ierr);
/* Create cell partition - We need to rewrite to use IS, use the MatPartition stuff */
ierr = PetscLogEventBegin(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
if (overlap > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Overlap > 1 not yet implemented");
ierr = DMPlexCreatePartition(dm, partitioner, height, overlap > 0 ? PETSC_TRUE : PETSC_FALSE, &cellPartSection, &cellPart, &origCellPartSection, &origCellPart);CHKERRQ(ierr);
/* Create SF assuming a serial partition for all processes: Could check for IS length here */
if (!rank) numRemoteRanks = numProcs;
else numRemoteRanks = 0;
ierr = PetscMalloc1(numRemoteRanks, &remoteRanks);CHKERRQ(ierr);
for (p = 0; p < numRemoteRanks; ++p) {
remoteRanks[p].rank = p;
remoteRanks[p].index = 0;
}
ierr = PetscSFCreate(comm, &partSF);CHKERRQ(ierr);
ierr = PetscSFSetGraph(partSF, 1, numRemoteRanks, NULL, PETSC_OWN_POINTER, remoteRanks, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-partition_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(cellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(cellPart, NULL);CHKERRQ(ierr);
if (origCellPart) {
ierr = PetscPrintf(comm, "Original Cell Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(origCellPartSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(origCellPart, NULL);CHKERRQ(ierr);
}
ierr = PetscSFView(partSF, NULL);CHKERRQ(ierr);
}
/* Close the partition over the mesh */
ierr = DMPlexCreatePartitionClosure(dm, cellPartSection, cellPart, &partSection, &part);CHKERRQ(ierr);
ierr = ISDestroy(&cellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&cellPartSection);CHKERRQ(ierr);
/* Create new mesh */
ierr = DMPlexCreate(comm, dmParallel);CHKERRQ(ierr);
ierr = DMPlexSetDimension(*dmParallel, dim);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dmParallel, "Parallel Mesh");CHKERRQ(ierr);
pmesh = (DM_Plex*) (*dmParallel)->data;
/* Distribute sieve points and the global point numbering (replaces creating remote bases) */
ierr = PetscSFConvertPartition(partSF, partSection, part, &renumbering, &pointSF);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Point Partition:\n");CHKERRQ(ierr);
ierr = PetscSectionView(partSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISView(part, NULL);CHKERRQ(ierr);
ierr = PetscSFView(pointSF, NULL);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Point Renumbering after partition:\n");CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(renumbering, NULL);CHKERRQ(ierr);
}
ierr = PetscLogEventEnd(DMPLEX_Partition,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscLogEventBegin(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Distribute cone section */
ierr = DMPlexGetConeSection(dm, &originalConeSection);CHKERRQ(ierr);
ierr = DMPlexGetConeSection(*dmParallel, &newConeSection);CHKERRQ(ierr);
ierr = PetscSFDistributeSection(pointSF, originalConeSection, &remoteOffsets, newConeSection);CHKERRQ(ierr);
ierr = DMSetUp(*dmParallel);CHKERRQ(ierr);
{
PetscInt pStart, pEnd, p;
ierr = PetscSectionGetChart(newConeSection, &pStart, &pEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt coneSize;
ierr = PetscSectionGetDof(newConeSection, p, &coneSize);CHKERRQ(ierr);
pmesh->maxConeSize = PetscMax(pmesh->maxConeSize, coneSize);
}
}
/* Communicate and renumber cones */
ierr = PetscSFCreateSectionSF(pointSF, originalConeSection, remoteOffsets, newConeSection, &coneSF);CHKERRQ(ierr);
ierr = DMPlexGetCones(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetCones(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(newConeSection, &newConesSize);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyBlock(renumbering, IS_GTOLM_MASK, newConesSize, newCones, NULL, newCones);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->prefix, "-cones_view", &flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(comm, "Serial Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(originalConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Parallel Cone Section:\n");CHKERRQ(ierr);
ierr = PetscSectionView(newConeSection, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscSFView(coneSF, NULL);CHKERRQ(ierr);
}
ierr = DMPlexGetConeOrientations(dm, &cones);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientations(*dmParallel, &newCones);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(coneSF, MPIU_INT, cones, newCones);CHKERRQ(ierr);
ierr = PetscSFDestroy(&coneSF);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_DistributeCones,dm,0,0,0);CHKERRQ(ierr);
/* Create supports and stratify sieve */
{
PetscInt pStart, pEnd;
ierr = PetscSectionGetChart(pmesh->coneSection, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionSetChart(pmesh->supportSection, pStart, pEnd);CHKERRQ(ierr);
}
ierr = DMPlexSymmetrize(*dmParallel);CHKERRQ(ierr);
ierr = DMPlexStratify(*dmParallel);CHKERRQ(ierr);
/* Distribute Coordinates */
{
PetscSection originalCoordSection, newCoordSection;
Vec originalCoordinates, newCoordinates;
PetscInt bs;
const char *name;
ierr = DMGetCoordinateSection(dm, &originalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(*dmParallel, &newCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &originalCoordinates);CHKERRQ(ierr);
ierr = VecCreate(comm, &newCoordinates);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject) originalCoordinates, &name);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) newCoordinates, name);CHKERRQ(ierr);
ierr = DMPlexDistributeField(dm, pointSF, originalCoordSection, originalCoordinates, newCoordSection, newCoordinates);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dmParallel, newCoordinates);CHKERRQ(ierr);
ierr = VecGetBlockSize(originalCoordinates, &bs);CHKERRQ(ierr);
ierr = VecSetBlockSize(newCoordinates, bs);CHKERRQ(ierr);
ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
}
/* Distribute labels */
ierr = PetscLogEventBegin(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
{
DMLabel next = mesh->labels, newNext = pmesh->labels;
PetscInt numLabels = 0, l;
/* Bcast number of labels */
while (next) {++numLabels; next = next->next;}
ierr = MPI_Bcast(&numLabels, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
next = mesh->labels;
for (l = 0; l < numLabels; ++l) {
DMLabel labelNew;
PetscBool isdepth;
/* Skip "depth" because it is recreated */
if (!rank) {ierr = PetscStrcmp(next->name, "depth", &isdepth);CHKERRQ(ierr);}
ierr = MPI_Bcast(&isdepth, 1, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
if (isdepth) {if (!rank) next = next->next; continue;}
ierr = DMLabelDistribute(next, partSection, part, renumbering, &labelNew);CHKERRQ(ierr);
/* Insert into list */
if (newNext) newNext->next = labelNew;
else pmesh->labels = labelNew;
newNext = labelNew;
if (!rank) next = next->next;
}
}
ierr = PetscLogEventEnd(DMPLEX_DistributeLabels,dm,0,0,0);CHKERRQ(ierr);
/* Setup hybrid structure */
{
const PetscInt *gpoints;
PetscInt depth, n, d;
for (d = 0; d <= dim; ++d) {pmesh->hybridPointMax[d] = mesh->hybridPointMax[d];}
ierr = MPI_Bcast(pmesh->hybridPointMax, dim+1, MPIU_INT, 0, comm);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetSize(renumbering, &n);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(renumbering, &gpoints);CHKERRQ(ierr);
ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
for (d = 0; d <= dim; ++d) {
PetscInt pmax = pmesh->hybridPointMax[d], newmax = 0, pEnd, stratum[2], p;
if (pmax < 0) continue;
ierr = DMPlexGetDepthStratum(dm, d > depth ? depth : d, &stratum[0], &stratum[1]);CHKERRQ(ierr);
ierr = DMPlexGetDepthStratum(*dmParallel, d, NULL, &pEnd);CHKERRQ(ierr);
ierr = MPI_Bcast(stratum, 2, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < n; ++p) {
const PetscInt point = gpoints[p];
if ((point >= stratum[0]) && (point < stratum[1]) && (point >= pmax)) ++newmax;
}
if (newmax > 0) pmesh->hybridPointMax[d] = pEnd - newmax;
else pmesh->hybridPointMax[d] = -1;
}
ierr = ISLocalToGlobalMappingRestoreIndices(renumbering, &gpoints);CHKERRQ(ierr);
}
/* Cleanup Partition */
ierr = ISLocalToGlobalMappingDestroy(&renumbering);CHKERRQ(ierr);
ierr = PetscSFDestroy(&partSF);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&partSection);CHKERRQ(ierr);
ierr = ISDestroy(&part);CHKERRQ(ierr);
/* Create point SF for parallel mesh */
ierr = PetscLogEventBegin(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
{
const PetscInt *leaves;
PetscSFNode *remotePoints, *rowners, *lowners;
PetscInt numRoots, numLeaves, numGhostPoints = 0, p, gp, *ghostPoints;
PetscInt pStart, pEnd;
ierr = DMPlexGetChart(*dmParallel, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSFGetGraph(pointSF, &numRoots, &numLeaves, &leaves, NULL);CHKERRQ(ierr);
ierr = PetscMalloc2(numRoots,&rowners,numLeaves,&lowners);CHKERRQ(ierr);
for (p=0; p<numRoots; p++) {
rowners[p].rank = -1;
rowners[p].index = -1;
}
if (origCellPart) {
/* Make sure points in the original partition are not assigned to other procs */
const PetscInt *origPoints;
ierr = DMPlexCreatePartitionClosure(dm, origCellPartSection, origCellPart, &origPartSection, &origPart);CHKERRQ(ierr);
ierr = ISGetIndices(origPart, &origPoints);CHKERRQ(ierr);
for (p = 0; p < numProcs; ++p) {
PetscInt dof, off, d;
ierr = PetscSectionGetDof(origPartSection, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(origPartSection, p, &off);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
rowners[origPoints[d]].rank = p;
}
}
ierr = ISRestoreIndices(origPart, &origPoints);CHKERRQ(ierr);
ierr = ISDestroy(&origPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origPartSection);CHKERRQ(ierr);
}
ierr = ISDestroy(&origCellPart);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&origCellPartSection);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].rank == rank) { /* Either put in a bid or we know we own it */
lowners[p].rank = rank;
lowners[p].index = leaves ? leaves[p] : p;
} else if (lowners[p].rank >= 0) { /* Point already claimed so flag so that MAXLOC does not listen to us */
lowners[p].rank = -2;
lowners[p].index = -2;
}
}
for (p=0; p<numRoots; p++) { /* Root must not participate in the rediction, flag so that MAXLOC does not use */
rowners[p].rank = -3;
rowners[p].index = -3;
}
ierr = PetscSFReduceBegin(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(pointSF, MPIU_2INT, lowners, rowners, MPI_MAXLOC);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(pointSF, MPIU_2INT, rowners, lowners);CHKERRQ(ierr);
for (p = 0; p < numLeaves; ++p) {
if (lowners[p].rank < 0 || lowners[p].index < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Cell partition corrupt: point not claimed");
if (lowners[p].rank != rank) ++numGhostPoints;
}
ierr = PetscMalloc1(numGhostPoints, &ghostPoints);CHKERRQ(ierr);
ierr = PetscMalloc1(numGhostPoints, &remotePoints);CHKERRQ(ierr);
for (p = 0, gp = 0; p < numLeaves; ++p) {
if (lowners[p].rank != rank) {
ghostPoints[gp] = leaves ? leaves[p] : p;
remotePoints[gp].rank = lowners[p].rank;
remotePoints[gp].index = lowners[p].index;
++gp;
}
}
ierr = PetscFree2(rowners,lowners);CHKERRQ(ierr);
ierr = PetscSFSetGraph((*dmParallel)->sf, pEnd - pStart, numGhostPoints, ghostPoints, PETSC_OWN_POINTER, remotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions((*dmParallel)->sf);CHKERRQ(ierr);
}
pmesh->useCone = mesh->useCone;
pmesh->useClosure = mesh->useClosure;
ierr = PetscLogEventEnd(DMPLEX_DistributeSF,dm,0,0,0);CHKERRQ(ierr);
/* Copy BC */
ierr = DMPlexCopyBoundary(dm, *dmParallel);CHKERRQ(ierr);
/* Cleanup */
if (sf) {*sf = pointSF;}
else {ierr = PetscSFDestroy(&pointSF);CHKERRQ(ierr);}
ierr = DMSetFromOptions(*dmParallel);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Distribute,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode private_VecView_Swarm_XDMF(Vec x,PetscViewer viewer)
{
long int *bytes = NULL;
PetscContainer container = NULL;
const char *viewername;
char datafile[PETSC_MAX_PATH_LEN];
PetscViewer fviewer;
PetscInt N,bs;
const char *vecname;
char fieldname[PETSC_MAX_PATH_LEN];
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)viewer,"XDMFViewerContext",(PetscObject*)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void**)&bytes);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)viewer),PETSC_ERR_SUP,"Valid to find attached data XDMFViewerContext");
ierr = PetscViewerFileGetName(viewer,&viewername);CHKERRQ(ierr);
ierr = private_CreateDataFileNameXDMF(viewername,datafile);CHKERRQ(ierr);
/* re-open a sub-viewer for all data fields */
/* name is viewer.name + "_swarm_fields.pbin" */
ierr = PetscViewerCreate(PetscObjectComm((PetscObject)viewer),&fviewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(fviewer,PETSCVIEWERBINARY);CHKERRQ(ierr);
ierr = PetscViewerBinarySetSkipHeader(fviewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerBinarySetSkipInfo(fviewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerFileSetMode(fviewer,FILE_MODE_APPEND);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(fviewer,datafile);CHKERRQ(ierr);
ierr = VecGetSize(x,&N);CHKERRQ(ierr);
ierr = VecGetBlockSize(x,&bs);CHKERRQ(ierr);
N = N/bs;
ierr = PetscObjectGetName((PetscObject)x,&vecname);CHKERRQ(ierr);
if (!vecname) {
ierr = PetscSNPrintf(fieldname,PETSC_MAX_PATH_LEN-1,"swarmfield_%D",((PetscObject)x)->tag);CHKERRQ(ierr);
} else {
ierr = PetscSNPrintf(fieldname,PETSC_MAX_PATH_LEN-1,"%s",vecname);CHKERRQ(ierr);
}
/* write data header */
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"<Attribute Center=\"Node\" Name=\"%s\" Type=\"None\">\n",fieldname);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
if (bs == 1) {
ierr = PetscViewerASCIIPrintf(viewer,"<DataItem Format=\"Binary\" Endian=\"Big\" DataType=\"Float\" Precision=\"8\" Dimensions=\"%D\" Seek=\"%D\">\n",N,bytes[0]);CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"<DataItem Format=\"Binary\" Endian=\"Big\" DataType=\"Float\" Precision=\"8\" Dimensions=\"%D %D\" Seek=\"%D\">\n",N,bs,bytes[0]);CHKERRQ(ierr);
}
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%s\n",datafile);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</DataItem>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</Attribute>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
/* write data */
ierr = VecView(x,fviewer);CHKERRQ(ierr);
bytes[0] += sizeof(PetscReal) * N * bs;
ierr = PetscViewerDestroy(&fviewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode VecAssemblyBegin_MPI_BTS(Vec X)
{
Vec_MPI *x = (Vec_MPI*)X->data;
PetscErrorCode ierr;
MPI_Comm comm;
PetscInt i,j,jb,bs;
PetscFunctionBegin;
if (X->stash.donotstash) PetscFunctionReturn(0);
ierr = PetscObjectGetComm((PetscObject)X,&comm);CHKERRQ(ierr);
ierr = VecGetBlockSize(X,&bs);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
{
InsertMode addv;
ierr = MPIU_Allreduce((PetscEnum*)&X->stash.insertmode,(PetscEnum*)&addv,1,MPIU_ENUM,MPI_BOR,comm);CHKERRQ(ierr);
if (addv == (ADD_VALUES|INSERT_VALUES)) SETERRQ(comm,PETSC_ERR_ARG_NOTSAMETYPE,"Some processors inserted values while others added");
}
#endif
X->bstash.insertmode = X->stash.insertmode; /* Block stash implicitly tracks InsertMode of scalar stash */
ierr = VecStashSortCompress_Private(&X->stash);CHKERRQ(ierr);
ierr = VecStashSortCompress_Private(&X->bstash);CHKERRQ(ierr);
if (!x->sendranks) {
PetscMPIInt nowners,bnowners,*owners,*bowners;
PetscInt ntmp;
ierr = VecStashGetOwnerList_Private(&X->stash,X->map,&nowners,&owners);CHKERRQ(ierr);
ierr = VecStashGetOwnerList_Private(&X->bstash,X->map,&bnowners,&bowners);CHKERRQ(ierr);
ierr = PetscMergeMPIIntArray(nowners,owners,bnowners,bowners,&ntmp,&x->sendranks);CHKERRQ(ierr);
x->nsendranks = ntmp;
ierr = PetscFree(owners);CHKERRQ(ierr);
ierr = PetscFree(bowners);CHKERRQ(ierr);
ierr = PetscMalloc1(x->nsendranks,&x->sendhdr);CHKERRQ(ierr);
ierr = PetscCalloc1(x->nsendranks,&x->sendptrs);CHKERRQ(ierr);
}
for (i=0,j=0,jb=0; i<x->nsendranks; i++) {
PetscMPIInt rank = x->sendranks[i];
x->sendhdr[i].insertmode = X->stash.insertmode;
/* Initialize pointers for non-empty stashes the first time around. Subsequent assemblies with
* VEC_SUBSET_OFF_PROC_ENTRIES will leave the old pointers (dangling because the stash has been collected) when
* there is nothing new to send, so that size-zero messages get sent instead. */
x->sendhdr[i].count = 0;
if (X->stash.n) {
x->sendptrs[i].ints = &X->stash.idx[j];
x->sendptrs[i].scalars = &X->stash.array[j];
for ( ; j<X->stash.n && X->stash.idx[j] < X->map->range[rank+1]; j++) x->sendhdr[i].count++;
}
x->sendhdr[i].bcount = 0;
if (X->bstash.n) {
x->sendptrs[i].intb = &X->bstash.idx[jb];
x->sendptrs[i].scalarb = &X->bstash.array[jb*bs];
for ( ; jb<X->bstash.n && X->bstash.idx[jb]*bs < X->map->range[rank+1]; jb++) x->sendhdr[i].bcount++;
}
}
if (!x->segrecvint) {ierr = PetscSegBufferCreate(sizeof(PetscInt),1000,&x->segrecvint);CHKERRQ(ierr);}
if (!x->segrecvscalar) {ierr = PetscSegBufferCreate(sizeof(PetscScalar),1000,&x->segrecvscalar);CHKERRQ(ierr);}
if (!x->segrecvframe) {ierr = PetscSegBufferCreate(sizeof(VecAssemblyFrame),50,&x->segrecvframe);CHKERRQ(ierr);}
if (x->recvhdr) { /* VEC_SUBSET_OFF_PROC_ENTRIES and this is not the first assembly */
PetscMPIInt tag[4];
if (!x->assembly_subset) SETERRQ(comm,PETSC_ERR_PLIB,"Attempt to reuse rendezvous when not VEC_SUBSET_OFF_PROC_ENTRIES");
for (i=0; i<4; i++) {ierr = PetscCommGetNewTag(comm,&tag[i]);CHKERRQ(ierr);}
for (i=0; i<x->nsendranks; i++) {
ierr = VecAssemblySend_MPI_Private(comm,tag,i,x->sendranks[i],x->sendhdr+i,x->sendreqs+4*i,X);CHKERRQ(ierr);
}
for (i=0; i<x->nrecvranks; i++) {
ierr = VecAssemblyRecv_MPI_Private(comm,tag,x->recvranks[i],x->recvhdr+i,x->recvreqs+4*i,X);CHKERRQ(ierr);
}
x->use_status = PETSC_TRUE;
} else { /* First time */
ierr = PetscCommBuildTwoSidedFReq(comm,3,MPIU_INT,x->nsendranks,x->sendranks,(PetscInt*)x->sendhdr,&x->nrecvranks,&x->recvranks,&x->recvhdr,4,&x->sendreqs,&x->recvreqs,VecAssemblySend_MPI_Private,VecAssemblyRecv_MPI_Private,X);CHKERRQ(ierr);
x->use_status = PETSC_FALSE;
}
{
PetscInt nstash,reallocs;
ierr = VecStashGetInfo_Private(&X->stash,&nstash,&reallocs);CHKERRQ(ierr);
ierr = PetscInfo2(X,"Stash has %D entries, uses %D mallocs.\n",nstash,reallocs);CHKERRQ(ierr);
ierr = VecStashGetInfo_Private(&X->bstash,&nstash,&reallocs);CHKERRQ(ierr);
ierr = PetscInfo2(X,"Block-Stash has %D entries, uses %D mallocs.\n",nstash,reallocs);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
