static PetscErrorCode ISCopy_Block(IS is,IS isy)
{
IS_Block *is_block = (IS_Block*)is->data,*isy_block = (IS_Block*)isy->data;
PetscInt bs, n, N, bsy, ny, Ny;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutGetBlockSize(is->map, &bs);
CHKERRQ(ierr);
ierr = PetscLayoutGetLocalSize(is->map, &n);
CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);
CHKERRQ(ierr);
ierr = PetscLayoutGetBlockSize(isy->map, &bsy);
CHKERRQ(ierr);
ierr = PetscLayoutGetLocalSize(isy->map, &ny);
CHKERRQ(ierr);
ierr = PetscLayoutGetSize(isy->map, &Ny);
CHKERRQ(ierr);
if (n != ny || N != Ny || bs != bsy) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Index sets incompatible");
isy_block->sorted = is_block->sorted;
ierr = PetscMemcpy(isy_block->idx,is_block->idx,(n/bs)*sizeof(PetscInt));
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ISCopy_General(IS is,IS isy)
{
IS_General *is_general = (IS_General*)is->data,*isy_general = (IS_General*)isy->data;
PetscInt n, N, ny, Ny;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr);
ierr = PetscLayoutGetLocalSize(isy->map, &ny);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(isy->map, &Ny);CHKERRQ(ierr);
if (n != ny || N != Ny) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Index sets incompatible");
isy_general->sorted = is_general->sorted;
ierr = PetscMemcpy(isy_general->idx,is_general->idx,n*sizeof(PetscInt));CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ISLocate_Block(IS is,PetscInt key,PetscInt *location)
{
IS_Block *sub = (IS_Block*)is->data;
PetscInt numIdx, i, bs, bkey, mkey;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutGetBlockSize(is->map,&bs);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map,&numIdx);CHKERRQ(ierr);
numIdx /= bs;
bkey = key / bs;
mkey = key % bs;
if (mkey < 0) {
bkey--;
mkey += bs;
}
if (sub->sorted) {
ierr = PetscFindInt(bkey,numIdx,sub->idx,location);CHKERRQ(ierr);
} else {
const PetscInt *idx = sub->idx;
*location = -1;
for (i = 0; i < numIdx; i++) {
if (idx[i] == bkey) {
*location = i;
break;
}
}
}
if (*location >= 0) {
*location = *location * bs + mkey;
}
PetscFunctionReturn(0);
}
PetscErrorCode DMSetUp_Composite(DM dm)
{
PetscErrorCode ierr;
PetscInt nprev = 0;
PetscMPIInt rank,size;
DM_Composite *com = (DM_Composite*)dm->data;
struct DMCompositeLink *next = com->next;
PetscLayout map;
PetscFunctionBegin;
if (com->setup) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Packer has already been setup");
ierr = PetscLayoutCreate(((PetscObject)dm)->comm,&map);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(map,com->n);CHKERRQ(ierr);
ierr = PetscLayoutSetSize(map,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(map,1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(map);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(map,&com->N);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(map,&com->rstart,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&map);CHKERRQ(ierr);
/* now set the rstart for each linked vector */
ierr = MPI_Comm_rank(((PetscObject)dm)->comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)dm)->comm,&size);CHKERRQ(ierr);
while (next) {
next->rstart = nprev;
nprev += next->n;
next->grstart = com->rstart + next->rstart;
ierr = PetscMalloc(size*sizeof(PetscInt),&next->grstarts);CHKERRQ(ierr);
ierr = MPI_Allgather(&next->grstart,1,MPIU_INT,next->grstarts,1,MPIU_INT,((PetscObject)dm)->comm);CHKERRQ(ierr);
next = next->next;
}
com->setup = PETSC_TRUE;
PetscFunctionReturn(0);
}
PetscErrorCode ISGetSize_General(IS is,PetscInt *size)
{
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutGetSize(is->map, size);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ISBlockGetSize_Block(IS is,PetscInt *size)
{
PetscInt bs, N;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutGetBlockSize(is->map, &bs);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr);
*size = N/bs;
PetscFunctionReturn(0);
}
PetscErrorCode ISView_General_Binary(IS is,PetscViewer viewer)
{
PetscErrorCode ierr;
IS_General *isa = (IS_General*) is->data;
PetscMPIInt rank,size,mesgsize,tag = ((PetscObject)viewer)->tag, mesglen;
PetscInt n,N,len,j,tr[2];
int fdes;
MPI_Status status;
PetscInt message_count,flowcontrolcount,*values;
PetscFunctionBegin;
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr);
/* determine maximum message to arrive */
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr);
tr[0] = IS_FILE_CLASSID;
tr[1] = N;
ierr = PetscViewerBinaryWrite(viewer,tr,2,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = MPI_Reduce(&n,&len,1,MPIU_INT,MPI_SUM,0,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlStart(viewer,&message_count,&flowcontrolcount);CHKERRQ(ierr);
if (!rank) {
ierr = PetscBinaryWrite(fdes,isa->idx,n,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscMalloc1(len,&values);CHKERRQ(ierr);
ierr = PetscMPIIntCast(len,&mesgsize);CHKERRQ(ierr);
/* receive and save messages */
for (j=1; j<size; j++) {
ierr = PetscViewerFlowControlStepMaster(viewer,j,&message_count,flowcontrolcount);CHKERRQ(ierr);
ierr = MPI_Recv(values,mesgsize,MPIU_INT,j,tag,PetscObjectComm((PetscObject)is),&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_INT,&mesglen);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fdes,values,(PetscInt)mesglen,PETSC_INT,PETSC_TRUE);CHKERRQ(ierr);
}
ierr = PetscViewerFlowControlEndMaster(viewer,&message_count);CHKERRQ(ierr);
ierr = PetscFree(values);CHKERRQ(ierr);
} else {
ierr = PetscViewerFlowControlStepWorker(viewer,rank,&message_count);CHKERRQ(ierr);
ierr = PetscMPIIntCast(n,&mesgsize);CHKERRQ(ierr);
ierr = MPI_Send(isa->idx,mesgsize,MPIU_INT,0,tag,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlEndWorker(viewer,&message_count);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode ISInvertPermutation_General(IS is,PetscInt nlocal,IS *isout)
{
IS_General *sub = (IS_General*)is->data;
PetscInt i,*ii,n,nstart;
const PetscInt *idx = sub->idx;
PetscMPIInt size;
IS istmp,nistmp;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr);
if (size == 1) {
ierr = PetscMalloc1(n,&ii);CHKERRQ(ierr);
for (i=0; i<n; i++) ii[idx[i]] = i;
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,ii,PETSC_OWN_POINTER,isout);CHKERRQ(ierr);
ierr = ISSetPermutation(*isout);CHKERRQ(ierr);
} else {
/* crude, nonscalable get entire IS on each processor */
if (nlocal == PETSC_DECIDE) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Do not yet support nlocal of PETSC_DECIDE");
ierr = ISAllGather(is,&istmp);CHKERRQ(ierr);
ierr = ISSetPermutation(istmp);CHKERRQ(ierr);
ierr = ISInvertPermutation(istmp,PETSC_DECIDE,&nistmp);CHKERRQ(ierr);
ierr = ISDestroy(&istmp);CHKERRQ(ierr);
/* get the part we need */
ierr = MPI_Scan(&nlocal,&nstart,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
{
PetscInt N;
PetscMPIInt rank;
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr);
if (rank == size-1) {
if (nstart != N) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Sum of nlocal lengths %d != total IS length %d",nstart,N);
}
}
#endif
nstart -= nlocal;
ierr = ISGetIndices(nistmp,&idx);CHKERRQ(ierr);
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)is),nlocal,idx+nstart,PETSC_COPY_VALUES,isout);CHKERRQ(ierr);
ierr = ISRestoreIndices(nistmp,&idx);CHKERRQ(ierr);
ierr = ISDestroy(&nistmp);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer)
{
MPI_Comm comm;
PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data;
FILE *fp;
PetscViewerVTKObjectLink link;
PetscSection coordSection, globalCoordSection;
PetscLayout vLayout;
Vec coordinates;
PetscReal lengthScale;
PetscInt vMax, totVertices, totCells;
PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr);
/* Vertices */
ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr);
if (vMax >= 0) {
PetscInt pStart, pEnd, p, localSize = 0;
ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr);
pEnd = PetscMin(pEnd, vMax);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof;
ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr);
if (dof > 0) ++localSize;
}
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr);
} else {
ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr);
}
ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr);
ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr);
/* Cells */
ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr);
/* Vertex fields */
for (link = vtk->link; link; link = link->next) {
if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE;
if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE;
}
if (hasPoint) {
ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection, newSection = NULL;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
DMLabel subpointMap, subpointMapX;
PetscInt dim, dimX, pStart, pEnd, qStart, qEnd;
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
/* Here is where we check whether dmX is a submesh of dm */
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr);
ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr);
if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) {
const PetscInt *ind = NULL;
IS subpointIS;
PetscInt n = 0, q;
ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr);
ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr);
if (subpointIS) {
ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr);
ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr);
}
ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr);
ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr);
for (q = qStart; q < qEnd; ++q) {
PetscInt dof, off, p;
ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr);
if (dof) {
ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr);
if (p >= pStart) {
ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr);
ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr);
}
}
}
if (subpointIS) {
ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr);
ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
}
/* No need to setup section */
section = newSection;
}
} else {
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr);
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);}
}
}
/* Cell Fields */
ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr);
if (hasCell || writePartition) {
ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr);
for (link = vtk->link; link; link = link->next) {
Vec X = (Vec) link->vec;
DM dmX;
PetscSection section, globalSection;
const char *name;
PetscInt enforceDof = PETSC_DETERMINE;
if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue;
if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3;
ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr);
ierr = VecGetDM(X, &dmX);CHKERRQ(ierr);
if (dmX) {
ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr);
} else {
PetscContainer c;
ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr);
if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr);
}
if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name);
ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr);
ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr);
}
if (writePartition) {
ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr);
ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr);
}
}
/* Cleanup */
ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr);
ierr = PetscFClose(comm, fp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ISView_General_Binary(IS is,PetscViewer viewer)
{
PetscErrorCode ierr;
PetscBool skipHeader,useMPIIO;
IS_General *isa = (IS_General*) is->data;
PetscMPIInt rank,size,mesgsize,tag = ((PetscObject)viewer)->tag, mesglen;
PetscInt n,N,len,j,tr[2];
int fdes;
MPI_Status status;
PetscInt message_count,flowcontrolcount,*values;
PetscFunctionBegin;
/* ierr = ISGetLayout(is,&map);CHKERRQ(ierr); */
ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr);
tr[0] = IS_FILE_CLASSID;
tr[1] = N;
ierr = PetscViewerBinaryGetSkipHeader(viewer,&skipHeader);CHKERRQ(ierr);
if (!skipHeader) {
ierr = PetscViewerBinaryWrite(viewer,tr,2,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
}
ierr = PetscViewerBinaryGetUseMPIIO(viewer,&useMPIIO);CHKERRQ(ierr);
#if defined(PETSC_HAVE_MPIIO)
if (useMPIIO) {
MPI_File mfdes;
MPI_Offset off;
PetscMPIInt lsize;
PetscInt rstart;
const PetscInt *iarray;
ierr = PetscMPIIntCast(n,&lsize);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIODescriptor(viewer,&mfdes);CHKERRQ(ierr);
ierr = PetscViewerBinaryGetMPIIOOffset(viewer,&off);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(is->map,&rstart,NULL);CHKERRQ(ierr);
off += rstart*(MPI_Offset)sizeof(PetscInt); /* off is MPI_Offset, not PetscMPIInt */
ierr = MPI_File_set_view(mfdes,off,MPIU_INT,MPIU_INT,(char*)"native",MPI_INFO_NULL);CHKERRQ(ierr);
ierr = ISGetIndices(is,&iarray);CHKERRQ(ierr);
ierr = MPIU_File_write_all(mfdes,(void*)iarray,lsize,MPIU_INT,MPI_STATUS_IGNORE);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&iarray);CHKERRQ(ierr);
ierr = PetscViewerBinaryAddMPIIOOffset(viewer,N*(MPI_Offset)sizeof(PetscInt));CHKERRQ(ierr);
PetscFunctionReturn(0);
}
#endif
ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr);
/* determine maximum message to arrive */
ierr = MPI_Reduce(&n,&len,1,MPIU_INT,MPI_MAX,0,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlStart(viewer,&message_count,&flowcontrolcount);CHKERRQ(ierr);
if (!rank) {
ierr = PetscBinaryWrite(fdes,isa->idx,n,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr);
ierr = PetscMalloc1(len,&values);CHKERRQ(ierr);
ierr = PetscMPIIntCast(len,&mesgsize);CHKERRQ(ierr);
/* receive and save messages */
for (j=1; j<size; j++) {
ierr = PetscViewerFlowControlStepMaster(viewer,j,&message_count,flowcontrolcount);CHKERRQ(ierr);
ierr = MPI_Recv(values,mesgsize,MPIU_INT,j,tag,PetscObjectComm((PetscObject)is),&status);CHKERRQ(ierr);
ierr = MPI_Get_count(&status,MPIU_INT,&mesglen);CHKERRQ(ierr);
ierr = PetscBinaryWrite(fdes,values,(PetscInt)mesglen,PETSC_INT,PETSC_TRUE);CHKERRQ(ierr);
}
ierr = PetscViewerFlowControlEndMaster(viewer,&message_count);CHKERRQ(ierr);
ierr = PetscFree(values);CHKERRQ(ierr);
} else {
ierr = PetscViewerFlowControlStepWorker(viewer,rank,&message_count);CHKERRQ(ierr);
ierr = PetscMPIIntCast(n,&mesgsize);CHKERRQ(ierr);
ierr = MPI_Send(isa->idx,mesgsize,MPIU_INT,0,tag,PetscObjectComm((PetscObject)is));CHKERRQ(ierr);
ierr = PetscViewerFlowControlEndWorker(viewer,&message_count);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode DMInterpolationSetUp(DMInterpolationInfo ctx, DM dm, PetscBool redundantPoints)
{
MPI_Comm comm = ctx->comm;
PetscScalar *a;
PetscInt p, q, i;
PetscMPIInt rank, size;
PetscErrorCode ierr;
Vec pointVec;
IS cellIS;
PetscLayout layout;
PetscReal *globalPoints;
PetscScalar *globalPointsScalar;
const PetscInt *ranges;
PetscMPIInt *counts, *displs;
const PetscInt *foundCells;
PetscMPIInt *foundProcs, *globalProcs;
PetscInt n, N;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
if (ctx->dim < 0) SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set");
/* Locate points */
n = ctx->nInput;
if (!redundantPoints) {
ierr = PetscLayoutCreate(comm, &layout);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(layout, n);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(layout);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(layout, &N);CHKERRQ(ierr);
/* Communicate all points to all processes */
ierr = PetscMalloc3(N*ctx->dim,&globalPoints,size,&counts,size,&displs);CHKERRQ(ierr);
ierr = PetscLayoutGetRanges(layout, &ranges);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
counts[p] = (ranges[p+1] - ranges[p])*ctx->dim;
displs[p] = ranges[p]*ctx->dim;
}
ierr = MPI_Allgatherv(ctx->points, n*ctx->dim, MPIU_REAL, globalPoints, counts, displs, MPIU_REAL, comm);CHKERRQ(ierr);
} else {
N = n;
globalPoints = ctx->points;
counts = displs = NULL;
layout = NULL;
}
#if 0
ierr = PetscMalloc3(N,&foundCells,N,&foundProcs,N,&globalProcs);CHKERRQ(ierr);
/* foundCells[p] = m->locatePoint(&globalPoints[p*ctx->dim]); */
#else
#if defined(PETSC_USE_COMPLEX)
ierr = PetscMalloc1(N,&globalPointsScalar);CHKERRQ(ierr);
for (i=0; i<N; i++) globalPointsScalar[i] = globalPoints[i];
#else
globalPointsScalar = globalPoints;
#endif
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, ctx->dim, N*ctx->dim, globalPointsScalar, &pointVec);CHKERRQ(ierr);
ierr = PetscMalloc2(N,&foundProcs,N,&globalProcs);CHKERRQ(ierr);
ierr = DMLocatePoints(dm, pointVec, &cellIS);CHKERRQ(ierr);
ierr = ISGetIndices(cellIS, &foundCells);CHKERRQ(ierr);
#endif
for (p = 0; p < N; ++p) {
if (foundCells[p] >= 0) foundProcs[p] = rank;
else foundProcs[p] = size;
}
/* Let the lowest rank process own each point */
ierr = MPI_Allreduce(foundProcs, globalProcs, N, MPI_INT, MPI_MIN, comm);CHKERRQ(ierr);
ctx->n = 0;
for (p = 0; p < N; ++p) {
if (globalProcs[p] == size) SETERRQ4(comm, PETSC_ERR_PLIB, "Point %d: %g %g %g not located in mesh", p, globalPoints[p*ctx->dim+0], ctx->dim > 1 ? globalPoints[p*ctx->dim+1] : 0.0, ctx->dim > 2 ? globalPoints[p*ctx->dim+2] : 0.0);
else if (globalProcs[p] == rank) ctx->n++;
}
/* Create coordinates vector and array of owned cells */
ierr = PetscMalloc1(ctx->n, &ctx->cells);CHKERRQ(ierr);
ierr = VecCreate(comm, &ctx->coords);CHKERRQ(ierr);
ierr = VecSetSizes(ctx->coords, ctx->n*ctx->dim, PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetBlockSize(ctx->coords, ctx->dim);CHKERRQ(ierr);
ierr = VecSetType(ctx->coords,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(ctx->coords, &a);CHKERRQ(ierr);
for (p = 0, q = 0, i = 0; p < N; ++p) {
if (globalProcs[p] == rank) {
PetscInt d;
for (d = 0; d < ctx->dim; ++d, ++i) a[i] = globalPoints[p*ctx->dim+d];
ctx->cells[q++] = foundCells[p];
}
}
ierr = VecRestoreArray(ctx->coords, &a);CHKERRQ(ierr);
#if 0
ierr = PetscFree3(foundCells,foundProcs,globalProcs);CHKERRQ(ierr);
#else
ierr = PetscFree2(foundProcs,globalProcs);CHKERRQ(ierr);
ierr = ISRestoreIndices(cellIS, &foundCells);CHKERRQ(ierr);
ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
ierr = VecDestroy(&pointVec);CHKERRQ(ierr);
#endif
if ((void*)globalPointsScalar != (void*)globalPoints) {ierr = PetscFree(globalPointsScalar);CHKERRQ(ierr);}
if (!redundantPoints) {ierr = PetscFree3(globalPoints,counts,displs);CHKERRQ(ierr);}
ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
