/*
DMSetVI - Marks a DM as associated with a VI problem. This causes the interpolation/restriction operators to
be restricted to only those variables NOT associated with active constraints.
*/
PetscErrorCode DMSetVI(DM dm,IS inactive)
{
PetscErrorCode ierr;
PetscContainer isnes;
DM_SNESVI *dmsnesvi;
PetscFunctionBegin;
if (!dm) PetscFunctionReturn(0);
ierr = PetscObjectReference((PetscObject)inactive);CHKERRQ(ierr);
ierr = PetscObjectQuery((PetscObject)dm,"VI",(PetscObject *)&isnes);CHKERRQ(ierr);
if (!isnes) {
ierr = PetscContainerCreate(((PetscObject)dm)->comm,&isnes);CHKERRQ(ierr);
ierr = PetscContainerSetUserDestroy(isnes,(PetscErrorCode (*)(void*))DMDestroy_SNESVI);CHKERRQ(ierr);
ierr = PetscNew(DM_SNESVI,&dmsnesvi);CHKERRQ(ierr);
ierr = PetscContainerSetPointer(isnes,(void*)dmsnesvi);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject)dm,"VI",(PetscObject)isnes);CHKERRQ(ierr);
ierr = PetscContainerDestroy(&isnes);CHKERRQ(ierr);
dmsnesvi->createinterpolation = dm->ops->createinterpolation;
dm->ops->createinterpolation = DMCreateInterpolation_SNESVI;
dmsnesvi->coarsen = dm->ops->coarsen;
dm->ops->coarsen = DMCoarsen_SNESVI;
dmsnesvi->createglobalvector = dm->ops->createglobalvector;
dm->ops->createglobalvector = DMCreateGlobalVector_SNESVI;
} else {
ierr = PetscContainerGetPointer(isnes,(void**)&dmsnesvi);CHKERRQ(ierr);
ierr = ISDestroy(&dmsnesvi->inactive);CHKERRQ(ierr);
}
ierr = DMClearGlobalVectors(dm);CHKERRQ(ierr);
ierr = ISGetLocalSize(inactive,&dmsnesvi->n);CHKERRQ(ierr);
dmsnesvi->inactive = inactive;
dmsnesvi->dm = dm;
PetscFunctionReturn(0);
}
PetscErrorCode private_PetscViewerDestroy_XDMF(PetscViewer *v)
{
PetscViewer viewer;
DM dm = NULL;
long int *bytes;
PetscContainer container = NULL;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!v) PetscFunctionReturn(0);
viewer = *v;
ierr = PetscObjectQuery((PetscObject)viewer,"DMSwarm",(PetscObject*)&dm);CHKERRQ(ierr);
if (dm) {
ierr = PetscViewerASCIIPrintf(viewer,"</Grid>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
}
/* close xdmf header */
ierr = PetscViewerASCIIPrintf(viewer,"</Domain>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</Xdmf>\n");CHKERRQ(ierr);
ierr = PetscObjectQuery((PetscObject)viewer,"XDMFViewerContext",(PetscObject*)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void**)&bytes);CHKERRQ(ierr);
ierr = PetscFree(bytes);CHKERRQ(ierr);
ierr = PetscContainerDestroy(&container);CHKERRQ(ierr);
}
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
*v = NULL;
PetscFunctionReturn(0);
}
PetscErrorCode MatMatMultNumeric_MPIAIJ_MPIAIJ(Mat A,Mat B,Mat C)
{
PetscErrorCode ierr;
Mat *seq;
Mat_MatMatMultMPI *mult;
PetscContainer container;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)C,"Mat_MatMatMultMPI",(PetscObject *)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void **)&mult);CHKERRQ(ierr);
} else {
SETERRQ(PETSC_ERR_PLIB,"Container does not exit");
}
seq = &mult->B_seq;
ierr = MatGetSubMatrices(B,1,&mult->isrowb,&mult->iscolb,MAT_REUSE_MATRIX,&seq);CHKERRQ(ierr);
mult->B_seq = *seq;
seq = &mult->A_loc;
ierr = MatGetSubMatrices(A,1,&mult->isrowa,&mult->isrowb,MAT_REUSE_MATRIX,&seq);CHKERRQ(ierr);
mult->A_loc = *seq;
ierr = MatMatMult_SeqAIJ_SeqAIJ(mult->A_loc,mult->B_seq,MAT_REUSE_MATRIX,0.0,&mult->C_seq);CHKERRQ(ierr);
ierr = PetscObjectReference((PetscObject)mult->C_seq);CHKERRQ(ierr);
ierr = MatMerge(((PetscObject)A)->comm,mult->C_seq,B->cmap->n,MAT_REUSE_MATRIX,&C);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
DMCreateInterpolation_SNESVI - Modifieds the interpolation obtained from the DM by removing all rows and columns associated with active constraints.
*/
PetscErrorCode DMCreateInterpolation_SNESVI(DM dm1,DM dm2,Mat *mat,Vec *vec)
{
PetscErrorCode ierr;
PetscContainer isnes;
DM_SNESVI *dmsnesvi1,*dmsnesvi2;
Mat interp;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)dm1,"VI",(PetscObject *)&isnes);CHKERRQ(ierr);
if (!isnes) SETERRQ(((PetscObject)dm1)->comm,PETSC_ERR_PLIB,"Composed VI data structure is missing");
ierr = PetscContainerGetPointer(isnes,(void**)&dmsnesvi1);CHKERRQ(ierr);
ierr = PetscObjectQuery((PetscObject)dm2,"VI",(PetscObject *)&isnes);CHKERRQ(ierr);
if (!isnes) SETERRQ(((PetscObject)dm2)->comm,PETSC_ERR_PLIB,"Composed VI data structure is missing");
ierr = PetscContainerGetPointer(isnes,(void**)&dmsnesvi2);CHKERRQ(ierr);
ierr = (*dmsnesvi1->createinterpolation)(dm1,dm2,&interp,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetSubMatrix(interp,dmsnesvi2->inactive,dmsnesvi1->inactive,MAT_INITIAL_MATRIX,mat);CHKERRQ(ierr);
ierr = MatDestroy(&interp);CHKERRQ(ierr);
*vec = 0;
PetscFunctionReturn(0);
}
/*
DMCreateGlobalVector_SNESVI - Creates global vector of the size of the reduced space
*/
PetscErrorCode DMCreateGlobalVector_SNESVI(DM dm,Vec *vec)
{
PetscErrorCode ierr;
PetscContainer isnes;
DM_SNESVI *dmsnesvi;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)dm,"VI",(PetscObject *)&isnes);CHKERRQ(ierr);
if (!isnes) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_PLIB,"Composed SNES is missing");
ierr = PetscContainerGetPointer(isnes,(void**)&dmsnesvi);CHKERRQ(ierr);
ierr = VecCreateMPI(((PetscObject)dm)->comm,dmsnesvi->n,PETSC_DETERMINE,vec);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatDestroy_SeqAIJ_RARt(Mat A)
{
PetscErrorCode ierr;
PetscContainer container;
Mat_RARt *rart=PETSC_NULL;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)A,"Mat_RARt",(PetscObject *)&container);CHKERRQ(ierr);
if (!container) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Container does not exit");
ierr = PetscContainerGetPointer(container,(void **)&rart);CHKERRQ(ierr);
A->ops->destroy = rart->destroy;
if (A->ops->destroy) {
ierr = (*A->ops->destroy)(A);CHKERRQ(ierr);
}
ierr = PetscObjectCompose((PetscObject)A,"Mat_RARt",0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatDestroy_MPIAIJ_MatMatMult(Mat A)
{
PetscErrorCode ierr;
PetscContainer container;
Mat_MatMatMultMPI *mult=PETSC_NULL;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)A,"Mat_MatMatMultMPI",(PetscObject *)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void **)&mult);CHKERRQ(ierr);
} else {
SETERRQ(PETSC_ERR_PLIB,"Container does not exit");
}
A->ops->destroy = mult->MatDestroy;
ierr = PetscObjectCompose((PetscObject)A,"Mat_MatMatMultMPI",0);CHKERRQ(ierr);
ierr = (*A->ops->destroy)(A);CHKERRQ(ierr);
ierr = PetscContainerDestroy(container);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatRARtNumeric_SeqAIJ_SeqAIJ(Mat A,Mat R,Mat C)
{
PetscErrorCode ierr;
Mat_RARt *rart;
PetscContainer container;
MatTransposeColoring matcoloring;
Mat Rt,RARt;
PetscLogDouble Mult_sp_den=0.0,app1=0.0,app2=0.0,t0,tf;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)C,"Mat_RARt",(PetscObject *)&container);CHKERRQ(ierr);
if (!container) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Container does not exit");
ierr = PetscContainerGetPointer(container,(void **)&rart);CHKERRQ(ierr);
/* Get dense Rt by Apply MatTransposeColoring to R */
matcoloring = rart->matcoloring;
Rt = rart->Rt;
ierr = PetscGetTime(&t0);CHKERRQ(ierr);
ierr = MatTransColoringApplySpToDen(matcoloring,R,Rt);CHKERRQ(ierr);
ierr = PetscGetTime(&tf);CHKERRQ(ierr);
app1 += tf - t0;
/* Get dense RARt = R*A*Rt */
ierr = PetscGetTime(&t0);CHKERRQ(ierr);
RARt = rart->RARt;
ierr = MatMatMatMultNumeric_SeqAIJ_SeqAIJ_SeqDense(R,A,Rt,RARt,rart->work);CHKERRQ(ierr);
ierr = PetscGetTime(&tf);CHKERRQ(ierr);
Mult_sp_den += tf - t0;
/* Recover C from C_dense */
ierr = PetscGetTime(&t0);CHKERRQ(ierr);
ierr = MatTransColoringApplyDenToSp(matcoloring,RARt,C);CHKERRQ(ierr);
ierr = PetscGetTime(&tf);CHKERRQ(ierr);
app2 += tf - t0;
#if defined(PETSC_USE_INFO)
ierr = PetscInfo4(C,"Num = ColorApp %g + %g + Mult_sp_den %g = %g\n",app1,app2,Mult_sp_den,app1+app2+Mult_sp_den);CHKERRQ(ierr);
#endif
PetscFunctionReturn(0);
}
PetscErrorCode MatDuplicate_MPIAIJ_MatMatMult(Mat A, MatDuplicateOption op, Mat *M)
{
PetscErrorCode ierr;
Mat_MatMatMultMPI *mult;
PetscContainer container;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)A,"Mat_MatMatMultMPI",(PetscObject *)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void **)&mult);CHKERRQ(ierr);
} else {
SETERRQ(PETSC_ERR_PLIB,"Container does not exit");
}
/* Note: the container is not duplicated, because it requires deep copying of
several large data sets (see PetscContainerDestroy_Mat_MatMatMultMPI()).
These data sets are only used for repeated calling of MatMatMultNumeric().
*M is unlikely being used in this way. Thus we create *M with pure mpiaij format */
ierr = (*mult->MatDuplicate)(A,op,M);CHKERRQ(ierr);
(*M)->ops->destroy = mult->MatDestroy; /* = MatDestroy_MPIAIJ, *M doesn't duplicate A's container! */
(*M)->ops->duplicate = mult->MatDuplicate; /* = MatDuplicate_MPIAIJ */
PetscFunctionReturn(0);
}
/*
obj - the PETSc object where the scalar pointer came from
base - the Fortran array address
addr - the Fortran offset from base
N - the amount of data
lx - the array space that is to be passed to XXXXRestoreArray()
*/
PetscErrorCode PetscScalarAddressFromFortran(PetscObject obj,PetscScalar *base,size_t addr,PetscInt N,PetscScalar **lx)
{
PetscErrorCode ierr;
PetscInt shift;
PetscContainer container;
PetscScalar *tlx;
ierr = PetscObjectQuery(obj,"GetArrayPtr",(PetscObject *)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void**)lx);CHKERRQ(ierr);
tlx = base + addr;
shift = *(PetscInt*)*lx;
ierr = PetscMemcpy(*lx,tlx,N*sizeof(PetscScalar));CHKERRQ(ierr);
tlx = (PetscScalar*)(((char *)tlx) - shift);
ierr = PetscFree(tlx);CHKERRQ(ierr);
ierr = PetscContainerDestroy(&container);CHKERRQ(ierr);
ierr = PetscObjectCompose(obj,"GetArrayPtr",0);CHKERRQ(ierr);
} else {
*lx = base + addr;
}
return 0;
}
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data;
FILE *fp;
PetscViewerVTKObjectLink link;
PetscSection coordSection, globalCoordSection;
PetscLayout vLayout;
Vec coordinates;
PetscReal lengthScale;
PetscInt vMax, totVertices, totCells;
PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr);
/* Vertices */
ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
if (vMax >= 0) {
PetscInt pStart, pEnd, p, localSize = 0;
ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr);
pEnd = PetscMin(pEnd, vMax);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr);
if (dof > 0) ++localSize;
}
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
} else {
ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr);
}
ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr);
ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr);
/* Cells */
ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr);
/* Vertex fields */
for (link = vtk->link; link; link = link->next) {
if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE;
if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE;
}
if (hasPoint) {
ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection, newSection = NULL;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
DMLabel subpointMap, subpointMapX;
PetscInt dim, dimX, pStart, pEnd, qStart, qEnd;
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
/* Here is where we check whether dmX is a submesh of dm */
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr);
if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) {
const PetscInt *ind = NULL;
IS subpointIS;
PetscInt n = 0, q;
ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr);
if (subpointIS) {
ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr);
ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr);
}
ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr);
for (q = qStart; q < qEnd; ++q) {
PetscInt dof, off, p;
ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr);
if (dof) {
ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr);
if (p >= pStart) {
ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr);
ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr);
}
}
}
if (subpointIS) {
ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr);
ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
}
/* No need to setup section */
section = newSection;
}
} else {
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr);
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);}
}
}
/* Cell Fields */
ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr);
if (hasCell || writePartition) {
ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
} else {
PetscContainer c;
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr);
if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
}
if (writePartition) {
ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr);
}
}
/* Cleanup */
ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
ierr = PetscFClose(comm, fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode private_DMSwarmView_XDMF(DM dm,PetscViewer viewer)
{
PetscBool isswarm = PETSC_FALSE;
const char *viewername;
char datafile[PETSC_MAX_PATH_LEN];
PetscViewer fviewer;
PetscInt k,ng,dim;
Vec dvec;
long int *bytes = NULL;
PetscContainer container = NULL;
const char *dmname;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)viewer,"XDMFViewerContext",(PetscObject*)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void**)&bytes);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)viewer),PETSC_ERR_SUP,"Valid to find attached data XDMFViewerContext");
ierr = PetscObjectTypeCompare((PetscObject)dm,DMSWARM,&isswarm);CHKERRQ(ierr);
if (!isswarm) SETERRQ(PetscObjectComm((PetscObject)viewer),PETSC_ERR_SUP,"Only valid for DMSwarm");
ierr = PetscObjectCompose((PetscObject)viewer,"DMSwarm",(PetscObject)dm);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscObjectGetName((PetscObject)dm,&dmname);CHKERRQ(ierr);
if (!dmname) {
ierr = DMGetOptionsPrefix(dm,&dmname);CHKERRQ(ierr);
}
if (!dmname) {
ierr = PetscViewerASCIIPrintf(viewer,"<Grid Name=\"DMSwarm\" GridType=\"Uniform\">\n");CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"<Grid Name=\"DMSwarm[%s]\" GridType=\"Uniform\">\n",dmname);CHKERRQ(ierr);
}
/* create a sub-viewer for topology, geometry and all data fields */
/* name is viewer.name + "_swarm_fields.pbin" */
ierr = PetscViewerCreate(PetscObjectComm((PetscObject)viewer),&fviewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(fviewer,PETSCVIEWERBINARY);CHKERRQ(ierr);
ierr = PetscViewerBinarySetSkipHeader(fviewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerBinarySetSkipInfo(fviewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerFileSetMode(fviewer,FILE_MODE_WRITE);CHKERRQ(ierr);
ierr = PetscViewerFileGetName(viewer,&viewername);CHKERRQ(ierr);
ierr = private_CreateDataFileNameXDMF(viewername,datafile);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(fviewer,datafile);CHKERRQ(ierr);
ierr = DMSwarmGetSize(dm,&ng);CHKERRQ(ierr);
/* write topology header */
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"<Topology Dimensions=\"%D\" TopologyType=\"Mixed\">\n",ng);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"<DataItem Format=\"Binary\" Endian=\"Big\" DataType=\"Int\" Dimensions=\"%D\" Seek=\"%D\">\n",ng*3,bytes[0]);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%s\n",datafile);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</DataItem>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</Topology>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
/* write topology data */
for (k=0; k<ng; k++) {
PetscInt pvertex[3];
pvertex[0] = 1;
pvertex[1] = 1;
pvertex[2] = k;
ierr = PetscViewerBinaryWrite(fviewer,pvertex,3,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
}
bytes[0] += sizeof(PetscInt) * ng * 3;
/* write geometry header */
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = DMGetDimension(dm,&dim);CHKERRQ(ierr);
switch (dim) {
case 1:
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"No support for 1D");
break;
case 2:
ierr = PetscViewerASCIIPrintf(viewer,"<Geometry Type=\"XY\">\n");CHKERRQ(ierr);
break;
case 3:
ierr = PetscViewerASCIIPrintf(viewer,"<Geometry Type=\"XYZ\">\n");CHKERRQ(ierr);
break;
}
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"<DataItem Format=\"Binary\" Endian=\"Big\" DataType=\"Float\" Precision=\"8\" Dimensions=\"%D %D\" Seek=\"%D\">\n",ng,dim,bytes[0]);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%s\n",datafile);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</DataItem>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</Geometry>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
/* write geometry data */
ierr = DMSwarmCreateGlobalVectorFromField(dm,DMSwarmPICField_coor,&dvec);CHKERRQ(ierr);
ierr = VecView(dvec,fviewer);CHKERRQ(ierr);
ierr = DMSwarmDestroyGlobalVectorFromField(dm,DMSwarmPICField_coor,&dvec);CHKERRQ(ierr);
bytes[0] += sizeof(PetscReal) * ng * dim;
ierr = PetscViewerDestroy(&fviewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode private_ISView_Swarm_XDMF(IS is,PetscViewer viewer)
{
long int *bytes = NULL;
PetscContainer container = NULL;
const char *viewername;
char datafile[PETSC_MAX_PATH_LEN];
PetscViewer fviewer;
PetscInt N,bs;
const char *vecname;
char fieldname[PETSC_MAX_PATH_LEN];
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)viewer,"XDMFViewerContext",(PetscObject*)&container);CHKERRQ(ierr);
if (container) {
ierr = PetscContainerGetPointer(container,(void**)&bytes);CHKERRQ(ierr);
} else SETERRQ(PetscObjectComm((PetscObject)viewer),PETSC_ERR_SUP,"Valid to find attached data XDMFViewerContext");
ierr = PetscViewerFileGetName(viewer,&viewername);CHKERRQ(ierr);
ierr = private_CreateDataFileNameXDMF(viewername,datafile);CHKERRQ(ierr);
/* re-open a sub-viewer for all data fields */
/* name is viewer.name + "_swarm_fields.pbin" */
ierr = PetscViewerCreate(PetscObjectComm((PetscObject)viewer),&fviewer);CHKERRQ(ierr);
ierr = PetscViewerSetType(fviewer,PETSCVIEWERBINARY);CHKERRQ(ierr);
ierr = PetscViewerBinarySetSkipHeader(fviewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerBinarySetSkipInfo(fviewer,PETSC_TRUE);CHKERRQ(ierr);
ierr = PetscViewerFileSetMode(fviewer,FILE_MODE_APPEND);CHKERRQ(ierr);
ierr = PetscViewerFileSetName(fviewer,datafile);CHKERRQ(ierr);
ierr = ISGetSize(is,&N);CHKERRQ(ierr);
ierr = ISGetBlockSize(is,&bs);CHKERRQ(ierr);
N = N/bs;
ierr = PetscObjectGetName((PetscObject)is,&vecname);CHKERRQ(ierr);
if (!vecname) {
ierr = PetscSNPrintf(fieldname,PETSC_MAX_PATH_LEN-1,"swarmfield_%D",((PetscObject)is)->tag);CHKERRQ(ierr);
} else {
ierr = PetscSNPrintf(fieldname,PETSC_MAX_PATH_LEN-1,"%s",vecname);CHKERRQ(ierr);
}
/* write data header */
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"<Attribute Center=\"Node\" Name=\"%s\" Type=\"None\">\n",fieldname);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
if (bs == 1) {
ierr = PetscViewerASCIIPrintf(viewer,"<DataItem Format=\"Binary\" Endian=\"Big\" DataType=\"Int\" Precision=\"4\" Dimensions=\"%D\" Seek=\"%D\">\n",N,bytes[0]);CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer,"<DataItem Format=\"Binary\" Endian=\"Big\" DataType=\"Int\" Precision=\"4\" Dimensions=\"%D %D\" Seek=\"%D\">\n",N,bs,bytes[0]);CHKERRQ(ierr);
}
ierr = PetscViewerASCIIPushTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%s\n",datafile);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</DataItem>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"</Attribute>\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPopTab(viewer);CHKERRQ(ierr);
/* write data */
ierr = ISView(is,fviewer);CHKERRQ(ierr);
bytes[0] += sizeof(PetscInt) * N * bs;
ierr = PetscViewerDestroy(&fviewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
DMCoarsen_SNESVI - Computes the regular coarsened DM then computes additional information about its inactive set
*/
PetscErrorCode DMCoarsen_SNESVI(DM dm1,MPI_Comm comm,DM *dm2)
{
PetscErrorCode ierr;
PetscContainer isnes;
DM_SNESVI *dmsnesvi1;
Vec finemarked,coarsemarked;
IS inactive;
VecScatter inject;
const PetscInt *index;
PetscInt n,k,cnt = 0,rstart,*coarseindex;
PetscScalar *marked;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)dm1,"VI",(PetscObject *)&isnes);CHKERRQ(ierr);
if (!isnes) SETERRQ(((PetscObject)dm1)->comm,PETSC_ERR_PLIB,"Composed VI data structure is missing");
ierr = PetscContainerGetPointer(isnes,(void**)&dmsnesvi1);CHKERRQ(ierr);
/* get the original coarsen */
ierr = (*dmsnesvi1->coarsen)(dm1,comm,dm2);CHKERRQ(ierr);
/* not sure why this extra reference is needed, but without the dm2 disappears too early */
ierr = PetscObjectReference((PetscObject)*dm2);CHKERRQ(ierr);
/* need to set back global vectors in order to use the original injection */
ierr = DMClearGlobalVectors(dm1);CHKERRQ(ierr);
dm1->ops->createglobalvector = dmsnesvi1->createglobalvector;
ierr = DMCreateGlobalVector(dm1,&finemarked);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(*dm2,&coarsemarked);CHKERRQ(ierr);
/*
fill finemarked with locations of inactive points
*/
ierr = ISGetIndices(dmsnesvi1->inactive,&index);CHKERRQ(ierr);
ierr = ISGetLocalSize(dmsnesvi1->inactive,&n);CHKERRQ(ierr);
ierr = VecSet(finemarked,0.0);CHKERRQ(ierr);
for (k=0;k<n;k++){
ierr = VecSetValue(finemarked,index[k],1.0,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(finemarked);CHKERRQ(ierr);
ierr = VecAssemblyEnd(finemarked);CHKERRQ(ierr);
ierr = DMCreateInjection(*dm2,dm1,&inject);CHKERRQ(ierr);
ierr = VecScatterBegin(inject,finemarked,coarsemarked,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(inject,finemarked,coarsemarked,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterDestroy(&inject);CHKERRQ(ierr);
/*
create index set list of coarse inactive points from coarsemarked
*/
ierr = VecGetLocalSize(coarsemarked,&n);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(coarsemarked,&rstart,PETSC_NULL);CHKERRQ(ierr);
ierr = VecGetArray(coarsemarked,&marked);CHKERRQ(ierr);
for (k=0; k<n; k++) {
if (marked[k] != 0.0) cnt++;
}
ierr = PetscMalloc(cnt*sizeof(PetscInt),&coarseindex);CHKERRQ(ierr);
cnt = 0;
for (k=0; k<n; k++) {
if (marked[k] != 0.0) coarseindex[cnt++] = k + rstart;
}
ierr = VecRestoreArray(coarsemarked,&marked);CHKERRQ(ierr);
ierr = ISCreateGeneral(((PetscObject)coarsemarked)->comm,cnt,coarseindex,PETSC_OWN_POINTER,&inactive);CHKERRQ(ierr);
ierr = DMClearGlobalVectors(dm1);CHKERRQ(ierr);
dm1->ops->createglobalvector = DMCreateGlobalVector_SNESVI;
ierr = DMSetVI(*dm2,inactive);CHKERRQ(ierr);
ierr = VecDestroy(&finemarked);CHKERRQ(ierr);
ierr = VecDestroy(&coarsemarked);CHKERRQ(ierr);
ierr = ISDestroy(&inactive);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
Performs an efficient scatter on the rows of B needed by this process; this is
a modification of the VecScatterBegin_() routines.
*/
PetscErrorCode MatMPIDenseScatter(Mat A,Mat B,Mat C,Mat *outworkB)
{
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)A->data;
PetscErrorCode ierr;
PetscScalar *b,*w,*svalues,*rvalues;
VecScatter ctx = aij->Mvctx;
VecScatter_MPI_General *from = (VecScatter_MPI_General*) ctx->fromdata;
VecScatter_MPI_General *to = ( VecScatter_MPI_General*) ctx->todata;
PetscInt i,j,k;
PetscInt *sindices,*sstarts,*rindices,*rstarts;
PetscMPIInt *sprocs,*rprocs,nrecvs;
MPI_Request *swaits,*rwaits;
MPI_Comm comm = ((PetscObject)A)->comm;
PetscMPIInt tag = ((PetscObject)ctx)->tag,ncols = B->cmap->N, nrows = aij->B->cmap->n,imdex,nrowsB = B->rmap->n;
MPI_Status status;
MPIAIJ_MPIDense *contents;
PetscContainer cont;
Mat workB;
PetscFunctionBegin;
ierr = PetscObjectQuery((PetscObject)C,"workB",(PetscObject*)&cont);CHKERRQ(ierr);
ierr = PetscContainerGetPointer(cont,(void**)&contents);CHKERRQ(ierr);
workB = *outworkB = contents->workB;
if (nrows != workB->rmap->n) SETERRQ2(PETSC_ERR_PLIB,"Number of rows of workB %D not equal to columns of aij->B %D",nrows,workB->cmap->n);
sindices = to->indices;
sstarts = to->starts;
sprocs = to->procs;
swaits = contents->swaits;
svalues = contents->svalues;
rindices = from->indices;
rstarts = from->starts;
rprocs = from->procs;
rwaits = contents->rwaits;
rvalues = contents->rvalues;
ierr = MatGetArray(B,&b);CHKERRQ(ierr);
ierr = MatGetArray(workB,&w);CHKERRQ(ierr);
for (i=0; i<from->n; i++) {
ierr = MPI_Irecv(rvalues+ncols*rstarts[i],ncols*(rstarts[i+1]-rstarts[i]),MPIU_SCALAR,rprocs[i],tag,comm,rwaits+i);CHKERRQ(ierr);
}
for (i=0; i<to->n; i++) {
/* pack a message at a time */
CHKMEMQ;
for (j=0; j<sstarts[i+1]-sstarts[i]; j++){
for (k=0; k<ncols; k++) {
svalues[ncols*(sstarts[i] + j) + k] = b[sindices[sstarts[i]+j] + nrowsB*k];
}
}
CHKMEMQ;
ierr = MPI_Isend(svalues+ncols*sstarts[i],ncols*(sstarts[i+1]-sstarts[i]),MPIU_SCALAR,sprocs[i],tag,comm,swaits+i);CHKERRQ(ierr);
}
nrecvs = from->n;
while (nrecvs) {
ierr = MPI_Waitany(from->n,rwaits,&imdex,&status);CHKERRQ(ierr);
nrecvs--;
/* unpack a message at a time */
CHKMEMQ;
for (j=0; j<rstarts[imdex+1]-rstarts[imdex]; j++){
for (k=0; k<ncols; k++) {
w[rindices[rstarts[imdex]+j] + nrows*k] = rvalues[ncols*(rstarts[imdex] + j) + k];
}
}
CHKMEMQ;
}
if (to->n) {ierr = MPI_Waitall(to->n,swaits,to->sstatus);CHKERRQ(ierr)}
