static PetscErrorCode ISCopy_Block(IS is,IS isy) { IS_Block *is_block = (IS_Block*)is->data,*isy_block = (IS_Block*)isy->data; PetscInt bs, n, N, bsy, ny, Ny; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLayoutGetBlockSize(is->map, &bs); CHKERRQ(ierr); ierr = PetscLayoutGetLocalSize(is->map, &n); CHKERRQ(ierr); ierr = PetscLayoutGetSize(is->map, &N); CHKERRQ(ierr); ierr = PetscLayoutGetBlockSize(isy->map, &bsy); CHKERRQ(ierr); ierr = PetscLayoutGetLocalSize(isy->map, &ny); CHKERRQ(ierr); ierr = PetscLayoutGetSize(isy->map, &Ny); CHKERRQ(ierr); if (n != ny || N != Ny || bs != bsy) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Index sets incompatible"); isy_block->sorted = is_block->sorted; ierr = PetscMemcpy(isy_block->idx,is_block->idx,(n/bs)*sizeof(PetscInt)); CHKERRQ(ierr); PetscFunctionReturn(0); }
static PetscErrorCode ISCopy_General(IS is,IS isy) { IS_General *is_general = (IS_General*)is->data,*isy_general = (IS_General*)isy->data; PetscInt n, N, ny, Ny; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr); ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr); ierr = PetscLayoutGetLocalSize(isy->map, &ny);CHKERRQ(ierr); ierr = PetscLayoutGetSize(isy->map, &Ny);CHKERRQ(ierr); if (n != ny || N != Ny) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Index sets incompatible"); isy_general->sorted = is_general->sorted; ierr = PetscMemcpy(isy_general->idx,is_general->idx,n*sizeof(PetscInt));CHKERRQ(ierr); PetscFunctionReturn(0); }
static PetscErrorCode ISLocate_Block(IS is,PetscInt key,PetscInt *location) { IS_Block *sub = (IS_Block*)is->data; PetscInt numIdx, i, bs, bkey, mkey; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLayoutGetBlockSize(is->map,&bs);CHKERRQ(ierr); ierr = PetscLayoutGetSize(is->map,&numIdx);CHKERRQ(ierr); numIdx /= bs; bkey = key / bs; mkey = key % bs; if (mkey < 0) { bkey--; mkey += bs; } if (sub->sorted) { ierr = PetscFindInt(bkey,numIdx,sub->idx,location);CHKERRQ(ierr); } else { const PetscInt *idx = sub->idx; *location = -1; for (i = 0; i < numIdx; i++) { if (idx[i] == bkey) { *location = i; break; } } } if (*location >= 0) { *location = *location * bs + mkey; } PetscFunctionReturn(0); }
PetscErrorCode DMSetUp_Composite(DM dm) { PetscErrorCode ierr; PetscInt nprev = 0; PetscMPIInt rank,size; DM_Composite *com = (DM_Composite*)dm->data; struct DMCompositeLink *next = com->next; PetscLayout map; PetscFunctionBegin; if (com->setup) SETERRQ(((PetscObject)dm)->comm,PETSC_ERR_ARG_WRONGSTATE,"Packer has already been setup"); ierr = PetscLayoutCreate(((PetscObject)dm)->comm,&map);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(map,com->n);CHKERRQ(ierr); ierr = PetscLayoutSetSize(map,PETSC_DETERMINE);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(map,1);CHKERRQ(ierr); ierr = PetscLayoutSetUp(map);CHKERRQ(ierr); ierr = PetscLayoutGetSize(map,&com->N);CHKERRQ(ierr); ierr = PetscLayoutGetRange(map,&com->rstart,PETSC_NULL);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&map);CHKERRQ(ierr); /* now set the rstart for each linked vector */ ierr = MPI_Comm_rank(((PetscObject)dm)->comm,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(((PetscObject)dm)->comm,&size);CHKERRQ(ierr); while (next) { next->rstart = nprev; nprev += next->n; next->grstart = com->rstart + next->rstart; ierr = PetscMalloc(size*sizeof(PetscInt),&next->grstarts);CHKERRQ(ierr); ierr = MPI_Allgather(&next->grstart,1,MPIU_INT,next->grstarts,1,MPIU_INT,((PetscObject)dm)->comm);CHKERRQ(ierr); next = next->next; } com->setup = PETSC_TRUE; PetscFunctionReturn(0); }
PetscErrorCode ISGetSize_General(IS is,PetscInt *size) { PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLayoutGetSize(is->map, size);CHKERRQ(ierr); PetscFunctionReturn(0); }
PetscErrorCode ISBlockGetSize_Block(IS is,PetscInt *size) { PetscInt bs, N; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLayoutGetBlockSize(is->map, &bs);CHKERRQ(ierr); ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr); *size = N/bs; PetscFunctionReturn(0); }
PetscErrorCode ISView_General_Binary(IS is,PetscViewer viewer) { PetscErrorCode ierr; IS_General *isa = (IS_General*) is->data; PetscMPIInt rank,size,mesgsize,tag = ((PetscObject)viewer)->tag, mesglen; PetscInt n,N,len,j,tr[2]; int fdes; MPI_Status status; PetscInt message_count,flowcontrolcount,*values; PetscFunctionBegin; ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr); ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr); ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr); /* determine maximum message to arrive */ ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr); tr[0] = IS_FILE_CLASSID; tr[1] = N; ierr = PetscViewerBinaryWrite(viewer,tr,2,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); ierr = MPI_Reduce(&n,&len,1,MPIU_INT,MPI_SUM,0,PetscObjectComm((PetscObject)is));CHKERRQ(ierr); ierr = PetscViewerFlowControlStart(viewer,&message_count,&flowcontrolcount);CHKERRQ(ierr); if (!rank) { ierr = PetscBinaryWrite(fdes,isa->idx,n,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); ierr = PetscMalloc1(len,&values);CHKERRQ(ierr); ierr = PetscMPIIntCast(len,&mesgsize);CHKERRQ(ierr); /* receive and save messages */ for (j=1; j<size; j++) { ierr = PetscViewerFlowControlStepMaster(viewer,j,&message_count,flowcontrolcount);CHKERRQ(ierr); ierr = MPI_Recv(values,mesgsize,MPIU_INT,j,tag,PetscObjectComm((PetscObject)is),&status);CHKERRQ(ierr); ierr = MPI_Get_count(&status,MPIU_INT,&mesglen);CHKERRQ(ierr); ierr = PetscBinaryWrite(fdes,values,(PetscInt)mesglen,PETSC_INT,PETSC_TRUE);CHKERRQ(ierr); } ierr = PetscViewerFlowControlEndMaster(viewer,&message_count);CHKERRQ(ierr); ierr = PetscFree(values);CHKERRQ(ierr); } else { ierr = PetscViewerFlowControlStepWorker(viewer,rank,&message_count);CHKERRQ(ierr); ierr = PetscMPIIntCast(n,&mesgsize);CHKERRQ(ierr); ierr = MPI_Send(isa->idx,mesgsize,MPIU_INT,0,tag,PetscObjectComm((PetscObject)is));CHKERRQ(ierr); ierr = PetscViewerFlowControlEndWorker(viewer,&message_count);CHKERRQ(ierr); } PetscFunctionReturn(0); }
PetscErrorCode ISInvertPermutation_General(IS is,PetscInt nlocal,IS *isout) { IS_General *sub = (IS_General*)is->data; PetscInt i,*ii,n,nstart; const PetscInt *idx = sub->idx; PetscMPIInt size; IS istmp,nistmp; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr); ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr); if (size == 1) { ierr = PetscMalloc1(n,&ii);CHKERRQ(ierr); for (i=0; i<n; i++) ii[idx[i]] = i; ierr = ISCreateGeneral(PETSC_COMM_SELF,n,ii,PETSC_OWN_POINTER,isout);CHKERRQ(ierr); ierr = ISSetPermutation(*isout);CHKERRQ(ierr); } else { /* crude, nonscalable get entire IS on each processor */ if (nlocal == PETSC_DECIDE) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Do not yet support nlocal of PETSC_DECIDE"); ierr = ISAllGather(is,&istmp);CHKERRQ(ierr); ierr = ISSetPermutation(istmp);CHKERRQ(ierr); ierr = ISInvertPermutation(istmp,PETSC_DECIDE,&nistmp);CHKERRQ(ierr); ierr = ISDestroy(&istmp);CHKERRQ(ierr); /* get the part we need */ ierr = MPI_Scan(&nlocal,&nstart,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)is));CHKERRQ(ierr); #if defined(PETSC_USE_DEBUG) { PetscInt N; PetscMPIInt rank; ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr); ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr); if (rank == size-1) { if (nstart != N) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Sum of nlocal lengths %d != total IS length %d",nstart,N); } } #endif nstart -= nlocal; ierr = ISGetIndices(nistmp,&idx);CHKERRQ(ierr); ierr = ISCreateGeneral(PetscObjectComm((PetscObject)is),nlocal,idx+nstart,PETSC_COPY_VALUES,isout);CHKERRQ(ierr); ierr = ISRestoreIndices(nistmp,&idx);CHKERRQ(ierr); ierr = ISDestroy(&nistmp);CHKERRQ(ierr); } PetscFunctionReturn(0); }
static PetscErrorCode DMPlexVTKWriteAll_ASCII(DM dm, PetscViewer viewer) { MPI_Comm comm; PetscViewer_VTK *vtk = (PetscViewer_VTK*) viewer->data; FILE *fp; PetscViewerVTKObjectLink link; PetscSection coordSection, globalCoordSection; PetscLayout vLayout; Vec coordinates; PetscReal lengthScale; PetscInt vMax, totVertices, totCells; PetscBool hasPoint = PETSC_FALSE, hasCell = PETSC_FALSE, writePartition = PETSC_FALSE; PetscErrorCode ierr; PetscFunctionBegin; ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr); ierr = PetscFOpen(comm, vtk->filename, "wb", &fp);CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "# vtk DataFile Version 2.0\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "Simplicial Mesh Example\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "ASCII\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "DATASET UNSTRUCTURED_GRID\n");CHKERRQ(ierr); /* Vertices */ ierr = DMPlexGetScale(dm, PETSC_UNIT_LENGTH, &lengthScale);CHKERRQ(ierr); ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr); ierr = PetscSectionCreateGlobalSection(coordSection, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalCoordSection);CHKERRQ(ierr); ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr); ierr = DMPlexGetHybridBounds(dm, NULL, NULL, NULL, &vMax);CHKERRQ(ierr); if (vMax >= 0) { PetscInt pStart, pEnd, p, localSize = 0; ierr = PetscSectionGetChart(globalCoordSection, &pStart, &pEnd);CHKERRQ(ierr); pEnd = PetscMin(pEnd, vMax); for (p = pStart; p < pEnd; ++p) { PetscInt dof; ierr = PetscSectionGetDof(globalCoordSection, p, &dof);CHKERRQ(ierr); if (dof > 0) ++localSize; } ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &vLayout);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(vLayout, localSize);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(vLayout, 1);CHKERRQ(ierr); ierr = PetscLayoutSetUp(vLayout);CHKERRQ(ierr); } else { ierr = PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalCoordSection, &vLayout);CHKERRQ(ierr); } ierr = PetscLayoutGetSize(vLayout, &totVertices);CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "POINTS %d double\n", totVertices);CHKERRQ(ierr); ierr = DMPlexVTKWriteSection_ASCII(dm, coordSection, globalCoordSection, coordinates, fp, 3, PETSC_DETERMINE, lengthScale);CHKERRQ(ierr); /* Cells */ ierr = DMPlexVTKWriteCells_ASCII(dm, fp, &totCells);CHKERRQ(ierr); /* Vertex fields */ for (link = vtk->link; link; link = link->next) { if ((link->ft == PETSC_VTK_POINT_FIELD) || (link->ft == PETSC_VTK_POINT_VECTOR_FIELD)) hasPoint = PETSC_TRUE; if ((link->ft == PETSC_VTK_CELL_FIELD) || (link->ft == PETSC_VTK_CELL_VECTOR_FIELD)) hasCell = PETSC_TRUE; } if (hasPoint) { ierr = PetscFPrintf(comm, fp, "POINT_DATA %d\n", totVertices);CHKERRQ(ierr); for (link = vtk->link; link; link = link->next) { Vec X = (Vec) link->vec; DM dmX; PetscSection section, globalSection, newSection = NULL; const char *name; PetscInt enforceDof = PETSC_DETERMINE; if ((link->ft != PETSC_VTK_POINT_FIELD) && (link->ft != PETSC_VTK_POINT_VECTOR_FIELD)) continue; if (link->ft == PETSC_VTK_POINT_VECTOR_FIELD) enforceDof = 3; ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr); ierr = VecGetDM(X, &dmX);CHKERRQ(ierr); if (dmX) { DMLabel subpointMap, subpointMapX; PetscInt dim, dimX, pStart, pEnd, qStart, qEnd; ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr); /* Here is where we check whether dmX is a submesh of dm */ ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr); ierr = DMGetDimension(dmX, &dimX);CHKERRQ(ierr); ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr); ierr = DMPlexGetChart(dmX, &qStart, &qEnd);CHKERRQ(ierr); ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr); ierr = DMPlexGetSubpointMap(dmX, &subpointMapX);CHKERRQ(ierr); if (((dim != dimX) || ((pEnd-pStart) < (qEnd-qStart))) && subpointMap && !subpointMapX) { const PetscInt *ind = NULL; IS subpointIS; PetscInt n = 0, q; ierr = PetscSectionGetChart(section, &qStart, &qEnd);CHKERRQ(ierr); ierr = DMPlexCreateSubpointIS(dm, &subpointIS);CHKERRQ(ierr); if (subpointIS) { ierr = ISGetLocalSize(subpointIS, &n);CHKERRQ(ierr); ierr = ISGetIndices(subpointIS, &ind);CHKERRQ(ierr); } ierr = PetscSectionCreate(comm, &newSection);CHKERRQ(ierr); ierr = PetscSectionSetChart(newSection, pStart, pEnd);CHKERRQ(ierr); for (q = qStart; q < qEnd; ++q) { PetscInt dof, off, p; ierr = PetscSectionGetDof(section, q, &dof);CHKERRQ(ierr); if (dof) { ierr = PetscFindInt(q, n, ind, &p);CHKERRQ(ierr); if (p >= pStart) { ierr = PetscSectionSetDof(newSection, p, dof);CHKERRQ(ierr); ierr = PetscSectionGetOffset(section, q, &off);CHKERRQ(ierr); ierr = PetscSectionSetOffset(newSection, p, off);CHKERRQ(ierr); } } } if (subpointIS) { ierr = ISRestoreIndices(subpointIS, &ind);CHKERRQ(ierr); ierr = ISDestroy(&subpointIS);CHKERRQ(ierr); } /* No need to setup section */ section = newSection; } } else { ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) §ion);CHKERRQ(ierr); if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name); } if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name); ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr); ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr); ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr); if (newSection) {ierr = PetscSectionDestroy(&newSection);CHKERRQ(ierr);} } } /* Cell Fields */ ierr = PetscOptionsGetBool(((PetscObject) dm)->prefix, "-dm_view_partition", &writePartition, NULL);CHKERRQ(ierr); if (hasCell || writePartition) { ierr = PetscFPrintf(comm, fp, "CELL_DATA %d\n", totCells);CHKERRQ(ierr); for (link = vtk->link; link; link = link->next) { Vec X = (Vec) link->vec; DM dmX; PetscSection section, globalSection; const char *name; PetscInt enforceDof = PETSC_DETERMINE; if ((link->ft != PETSC_VTK_CELL_FIELD) && (link->ft != PETSC_VTK_CELL_VECTOR_FIELD)) continue; if (link->ft == PETSC_VTK_CELL_VECTOR_FIELD) enforceDof = 3; ierr = PetscObjectGetName(link->vec, &name);CHKERRQ(ierr); ierr = VecGetDM(X, &dmX);CHKERRQ(ierr); if (dmX) { ierr = DMGetDefaultSection(dmX, §ion);CHKERRQ(ierr); } else { PetscContainer c; ierr = PetscObjectQuery(link->vec, "section", (PetscObject*) &c);CHKERRQ(ierr); if (!c) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name); ierr = PetscContainerGetPointer(c, (void**) §ion);CHKERRQ(ierr); } if (!section) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Vector %s had no PetscSection composed with it", name); ierr = PetscSectionCreateGlobalSection(section, dm->sf, PETSC_FALSE, PETSC_FALSE, &globalSection);CHKERRQ(ierr); ierr = DMPlexVTKWriteField_ASCII(dm, section, globalSection, X, name, fp, enforceDof, PETSC_DETERMINE, 1.0);CHKERRQ(ierr); ierr = PetscSectionDestroy(&globalSection);CHKERRQ(ierr); } if (writePartition) { ierr = PetscFPrintf(comm, fp, "SCALARS partition int 1\n");CHKERRQ(ierr); ierr = PetscFPrintf(comm, fp, "LOOKUP_TABLE default\n");CHKERRQ(ierr); ierr = DMPlexVTKWritePartition_ASCII(dm, fp);CHKERRQ(ierr); } } /* Cleanup */ ierr = PetscSectionDestroy(&globalCoordSection);CHKERRQ(ierr); ierr = PetscLayoutDestroy(&vLayout);CHKERRQ(ierr); ierr = PetscFClose(comm, fp);CHKERRQ(ierr); PetscFunctionReturn(0); }
static PetscErrorCode ISView_General_Binary(IS is,PetscViewer viewer) { PetscErrorCode ierr; PetscBool skipHeader,useMPIIO; IS_General *isa = (IS_General*) is->data; PetscMPIInt rank,size,mesgsize,tag = ((PetscObject)viewer)->tag, mesglen; PetscInt n,N,len,j,tr[2]; int fdes; MPI_Status status; PetscInt message_count,flowcontrolcount,*values; PetscFunctionBegin; /* ierr = ISGetLayout(is,&map);CHKERRQ(ierr); */ ierr = PetscLayoutGetLocalSize(is->map, &n);CHKERRQ(ierr); ierr = PetscLayoutGetSize(is->map, &N);CHKERRQ(ierr); tr[0] = IS_FILE_CLASSID; tr[1] = N; ierr = PetscViewerBinaryGetSkipHeader(viewer,&skipHeader);CHKERRQ(ierr); if (!skipHeader) { ierr = PetscViewerBinaryWrite(viewer,tr,2,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); } ierr = PetscViewerBinaryGetUseMPIIO(viewer,&useMPIIO);CHKERRQ(ierr); #if defined(PETSC_HAVE_MPIIO) if (useMPIIO) { MPI_File mfdes; MPI_Offset off; PetscMPIInt lsize; PetscInt rstart; const PetscInt *iarray; ierr = PetscMPIIntCast(n,&lsize);CHKERRQ(ierr); ierr = PetscViewerBinaryGetMPIIODescriptor(viewer,&mfdes);CHKERRQ(ierr); ierr = PetscViewerBinaryGetMPIIOOffset(viewer,&off);CHKERRQ(ierr); ierr = PetscLayoutGetRange(is->map,&rstart,NULL);CHKERRQ(ierr); off += rstart*(MPI_Offset)sizeof(PetscInt); /* off is MPI_Offset, not PetscMPIInt */ ierr = MPI_File_set_view(mfdes,off,MPIU_INT,MPIU_INT,(char*)"native",MPI_INFO_NULL);CHKERRQ(ierr); ierr = ISGetIndices(is,&iarray);CHKERRQ(ierr); ierr = MPIU_File_write_all(mfdes,(void*)iarray,lsize,MPIU_INT,MPI_STATUS_IGNORE);CHKERRQ(ierr); ierr = ISRestoreIndices(is,&iarray);CHKERRQ(ierr); ierr = PetscViewerBinaryAddMPIIOOffset(viewer,N*(MPI_Offset)sizeof(PetscInt));CHKERRQ(ierr); PetscFunctionReturn(0); } #endif ierr = PetscViewerBinaryGetDescriptor(viewer,&fdes);CHKERRQ(ierr); ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)is),&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PetscObjectComm((PetscObject)is),&size);CHKERRQ(ierr); /* determine maximum message to arrive */ ierr = MPI_Reduce(&n,&len,1,MPIU_INT,MPI_MAX,0,PetscObjectComm((PetscObject)is));CHKERRQ(ierr); ierr = PetscViewerFlowControlStart(viewer,&message_count,&flowcontrolcount);CHKERRQ(ierr); if (!rank) { ierr = PetscBinaryWrite(fdes,isa->idx,n,PETSC_INT,PETSC_FALSE);CHKERRQ(ierr); ierr = PetscMalloc1(len,&values);CHKERRQ(ierr); ierr = PetscMPIIntCast(len,&mesgsize);CHKERRQ(ierr); /* receive and save messages */ for (j=1; j<size; j++) { ierr = PetscViewerFlowControlStepMaster(viewer,j,&message_count,flowcontrolcount);CHKERRQ(ierr); ierr = MPI_Recv(values,mesgsize,MPIU_INT,j,tag,PetscObjectComm((PetscObject)is),&status);CHKERRQ(ierr); ierr = MPI_Get_count(&status,MPIU_INT,&mesglen);CHKERRQ(ierr); ierr = PetscBinaryWrite(fdes,values,(PetscInt)mesglen,PETSC_INT,PETSC_TRUE);CHKERRQ(ierr); } ierr = PetscViewerFlowControlEndMaster(viewer,&message_count);CHKERRQ(ierr); ierr = PetscFree(values);CHKERRQ(ierr); } else { ierr = PetscViewerFlowControlStepWorker(viewer,rank,&message_count);CHKERRQ(ierr); ierr = PetscMPIIntCast(n,&mesgsize);CHKERRQ(ierr); ierr = MPI_Send(isa->idx,mesgsize,MPIU_INT,0,tag,PetscObjectComm((PetscObject)is));CHKERRQ(ierr); ierr = PetscViewerFlowControlEndWorker(viewer,&message_count);CHKERRQ(ierr); } PetscFunctionReturn(0); }
PetscErrorCode DMInterpolationSetUp(DMInterpolationInfo ctx, DM dm, PetscBool redundantPoints) { MPI_Comm comm = ctx->comm; PetscScalar *a; PetscInt p, q, i; PetscMPIInt rank, size; PetscErrorCode ierr; Vec pointVec; IS cellIS; PetscLayout layout; PetscReal *globalPoints; PetscScalar *globalPointsScalar; const PetscInt *ranges; PetscMPIInt *counts, *displs; const PetscInt *foundCells; PetscMPIInt *foundProcs, *globalProcs; PetscInt n, N; PetscFunctionBegin; PetscValidHeaderSpecific(dm, DM_CLASSID, 1); ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr); if (ctx->dim < 0) SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set"); /* Locate points */ n = ctx->nInput; if (!redundantPoints) { ierr = PetscLayoutCreate(comm, &layout);CHKERRQ(ierr); ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr); ierr = PetscLayoutSetLocalSize(layout, n);CHKERRQ(ierr); ierr = PetscLayoutSetUp(layout);CHKERRQ(ierr); ierr = PetscLayoutGetSize(layout, &N);CHKERRQ(ierr); /* Communicate all points to all processes */ ierr = PetscMalloc3(N*ctx->dim,&globalPoints,size,&counts,size,&displs);CHKERRQ(ierr); ierr = PetscLayoutGetRanges(layout, &ranges);CHKERRQ(ierr); for (p = 0; p < size; ++p) { counts[p] = (ranges[p+1] - ranges[p])*ctx->dim; displs[p] = ranges[p]*ctx->dim; } ierr = MPI_Allgatherv(ctx->points, n*ctx->dim, MPIU_REAL, globalPoints, counts, displs, MPIU_REAL, comm);CHKERRQ(ierr); } else { N = n; globalPoints = ctx->points; counts = displs = NULL; layout = NULL; } #if 0 ierr = PetscMalloc3(N,&foundCells,N,&foundProcs,N,&globalProcs);CHKERRQ(ierr); /* foundCells[p] = m->locatePoint(&globalPoints[p*ctx->dim]); */ #else #if defined(PETSC_USE_COMPLEX) ierr = PetscMalloc1(N,&globalPointsScalar);CHKERRQ(ierr); for (i=0; i<N; i++) globalPointsScalar[i] = globalPoints[i]; #else globalPointsScalar = globalPoints; #endif ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, ctx->dim, N*ctx->dim, globalPointsScalar, &pointVec);CHKERRQ(ierr); ierr = PetscMalloc2(N,&foundProcs,N,&globalProcs);CHKERRQ(ierr); ierr = DMLocatePoints(dm, pointVec, &cellIS);CHKERRQ(ierr); ierr = ISGetIndices(cellIS, &foundCells);CHKERRQ(ierr); #endif for (p = 0; p < N; ++p) { if (foundCells[p] >= 0) foundProcs[p] = rank; else foundProcs[p] = size; } /* Let the lowest rank process own each point */ ierr = MPI_Allreduce(foundProcs, globalProcs, N, MPI_INT, MPI_MIN, comm);CHKERRQ(ierr); ctx->n = 0; for (p = 0; p < N; ++p) { if (globalProcs[p] == size) SETERRQ4(comm, PETSC_ERR_PLIB, "Point %d: %g %g %g not located in mesh", p, globalPoints[p*ctx->dim+0], ctx->dim > 1 ? globalPoints[p*ctx->dim+1] : 0.0, ctx->dim > 2 ? globalPoints[p*ctx->dim+2] : 0.0); else if (globalProcs[p] == rank) ctx->n++; } /* Create coordinates vector and array of owned cells */ ierr = PetscMalloc1(ctx->n, &ctx->cells);CHKERRQ(ierr); ierr = VecCreate(comm, &ctx->coords);CHKERRQ(ierr); ierr = VecSetSizes(ctx->coords, ctx->n*ctx->dim, PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetBlockSize(ctx->coords, ctx->dim);CHKERRQ(ierr); ierr = VecSetType(ctx->coords,VECSTANDARD);CHKERRQ(ierr); ierr = VecGetArray(ctx->coords, &a);CHKERRQ(ierr); for (p = 0, q = 0, i = 0; p < N; ++p) { if (globalProcs[p] == rank) { PetscInt d; for (d = 0; d < ctx->dim; ++d, ++i) a[i] = globalPoints[p*ctx->dim+d]; ctx->cells[q++] = foundCells[p]; } } ierr = VecRestoreArray(ctx->coords, &a);CHKERRQ(ierr); #if 0 ierr = PetscFree3(foundCells,foundProcs,globalProcs);CHKERRQ(ierr); #else ierr = PetscFree2(foundProcs,globalProcs);CHKERRQ(ierr); ierr = ISRestoreIndices(cellIS, &foundCells);CHKERRQ(ierr); ierr = ISDestroy(&cellIS);CHKERRQ(ierr); ierr = VecDestroy(&pointVec);CHKERRQ(ierr); #endif if ((void*)globalPointsScalar != (void*)globalPoints) {ierr = PetscFree(globalPointsScalar);CHKERRQ(ierr);} if (!redundantPoints) {ierr = PetscFree3(globalPoints,counts,displs);CHKERRQ(ierr);} ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr); PetscFunctionReturn(0); }