PetscErrorCode MatPartitioningHierarchical_ReassembleFineparts(Mat adj, IS fineparts, ISLocalToGlobalMapping mapping, IS *sfineparts)
{
PetscInt *local_indices, *global_indices,*owners,*sfineparts_indices,localsize,i;
const PetscInt *ranges,*fineparts_indices;
PetscMPIInt rank;
MPI_Comm comm;
PetscLayout rmap;
PetscSFNode *remote;
PetscSF sf;
PetscErrorCode ierr;
PetscFunctionBegin;
/*get communicator */
ierr = PetscObjectGetComm((PetscObject)adj,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MatGetLayouts(adj,&rmap,PETSC_NULL);CHKERRQ(ierr);
ierr = ISGetLocalSize(fineparts,&localsize);CHKERRQ(ierr);
ierr = PetscCalloc2(localsize,&global_indices,localsize,&local_indices);CHKERRQ(ierr);
for(i=0; i<localsize; i++){
local_indices[i] = i;
}
/*global indices */
ierr = ISLocalToGlobalMappingApply(mapping,localsize,local_indices,global_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(localsize,&owners);CHKERRQ(ierr);
/*find owners for global indices */
for(i=0; i<localsize; i++){
ierr = PetscLayoutFindOwner(rmap,global_indices[i],&owners[i]);CHKERRQ(ierr);
}
/*ranges */
ierr = PetscLayoutGetRanges(rmap,&ranges);CHKERRQ(ierr);
ierr = PetscCalloc1(ranges[rank+1]-ranges[rank],&sfineparts_indices);CHKERRQ(ierr);
ierr = ISGetIndices(fineparts,&fineparts_indices);CHKERRQ(ierr);
/*create a SF to exchange data */
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscCalloc1(localsize,&remote);CHKERRQ(ierr);
for(i=0; i<localsize; i++){
remote[i].rank = owners[i];
remote[i].index = global_indices[i]-ranges[owners[i]];
}
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
/*not sure how to add prefix to sf*/
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,localsize,localsize,PETSC_NULL,PETSC_OWN_POINTER,remote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sf,MPIU_INT,fineparts_indices,sfineparts_indices,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sf,MPIU_INT,fineparts_indices,sfineparts_indices,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = ISRestoreIndices(fineparts,&fineparts_indices);CHKERRQ(ierr);
/* comm self */
ierr = ISCreateGeneral(comm,ranges[rank+1]-ranges[rank],sfineparts_indices,PETSC_OWN_POINTER,sfineparts);CHKERRQ(ierr);
ierr = PetscFree2(global_indices,local_indices);CHKERRQ(ierr);
ierr = PetscFree(owners);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode DMInterpolationSetUp(DMInterpolationInfo ctx, DM dm, PetscBool redundantPoints)
{
MPI_Comm comm = ctx->comm;
PetscScalar *a;
PetscInt p, q, i;
PetscMPIInt rank, size;
PetscErrorCode ierr;
Vec pointVec;
IS cellIS;
PetscLayout layout;
PetscReal *globalPoints;
PetscScalar *globalPointsScalar;
const PetscInt *ranges;
PetscMPIInt *counts, *displs;
const PetscInt *foundCells;
PetscMPIInt *foundProcs, *globalProcs;
PetscInt n, N;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
if (ctx->dim < 0) SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "The spatial dimension has not been set");
/* Locate points */
n = ctx->nInput;
if (!redundantPoints) {
ierr = PetscLayoutCreate(comm, &layout);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(layout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(layout, n);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(layout);CHKERRQ(ierr);
ierr = PetscLayoutGetSize(layout, &N);CHKERRQ(ierr);
/* Communicate all points to all processes */
ierr = PetscMalloc3(N*ctx->dim,&globalPoints,size,&counts,size,&displs);CHKERRQ(ierr);
ierr = PetscLayoutGetRanges(layout, &ranges);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
counts[p] = (ranges[p+1] - ranges[p])*ctx->dim;
displs[p] = ranges[p]*ctx->dim;
}
ierr = MPI_Allgatherv(ctx->points, n*ctx->dim, MPIU_REAL, globalPoints, counts, displs, MPIU_REAL, comm);CHKERRQ(ierr);
} else {
N = n;
globalPoints = ctx->points;
counts = displs = NULL;
layout = NULL;
}
#if 0
ierr = PetscMalloc3(N,&foundCells,N,&foundProcs,N,&globalProcs);CHKERRQ(ierr);
/* foundCells[p] = m->locatePoint(&globalPoints[p*ctx->dim]); */
#else
#if defined(PETSC_USE_COMPLEX)
ierr = PetscMalloc1(N,&globalPointsScalar);CHKERRQ(ierr);
for (i=0; i<N; i++) globalPointsScalar[i] = globalPoints[i];
#else
globalPointsScalar = globalPoints;
#endif
ierr = VecCreateSeqWithArray(PETSC_COMM_SELF, ctx->dim, N*ctx->dim, globalPointsScalar, &pointVec);CHKERRQ(ierr);
ierr = PetscMalloc2(N,&foundProcs,N,&globalProcs);CHKERRQ(ierr);
ierr = DMLocatePoints(dm, pointVec, &cellIS);CHKERRQ(ierr);
ierr = ISGetIndices(cellIS, &foundCells);CHKERRQ(ierr);
#endif
for (p = 0; p < N; ++p) {
if (foundCells[p] >= 0) foundProcs[p] = rank;
else foundProcs[p] = size;
}
/* Let the lowest rank process own each point */
ierr = MPI_Allreduce(foundProcs, globalProcs, N, MPI_INT, MPI_MIN, comm);CHKERRQ(ierr);
ctx->n = 0;
for (p = 0; p < N; ++p) {
if (globalProcs[p] == size) SETERRQ4(comm, PETSC_ERR_PLIB, "Point %d: %g %g %g not located in mesh", p, globalPoints[p*ctx->dim+0], ctx->dim > 1 ? globalPoints[p*ctx->dim+1] : 0.0, ctx->dim > 2 ? globalPoints[p*ctx->dim+2] : 0.0);
else if (globalProcs[p] == rank) ctx->n++;
}
/* Create coordinates vector and array of owned cells */
ierr = PetscMalloc1(ctx->n, &ctx->cells);CHKERRQ(ierr);
ierr = VecCreate(comm, &ctx->coords);CHKERRQ(ierr);
ierr = VecSetSizes(ctx->coords, ctx->n*ctx->dim, PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetBlockSize(ctx->coords, ctx->dim);CHKERRQ(ierr);
ierr = VecSetType(ctx->coords,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(ctx->coords, &a);CHKERRQ(ierr);
for (p = 0, q = 0, i = 0; p < N; ++p) {
if (globalProcs[p] == rank) {
PetscInt d;
for (d = 0; d < ctx->dim; ++d, ++i) a[i] = globalPoints[p*ctx->dim+d];
ctx->cells[q++] = foundCells[p];
}
}
ierr = VecRestoreArray(ctx->coords, &a);CHKERRQ(ierr);
#if 0
ierr = PetscFree3(foundCells,foundProcs,globalProcs);CHKERRQ(ierr);
#else
ierr = PetscFree2(foundProcs,globalProcs);CHKERRQ(ierr);
ierr = ISRestoreIndices(cellIS, &foundCells);CHKERRQ(ierr);
ierr = ISDestroy(&cellIS);CHKERRQ(ierr);
ierr = VecDestroy(&pointVec);CHKERRQ(ierr);
#endif
if ((void*)globalPointsScalar != (void*)globalPoints) {ierr = PetscFree(globalPointsScalar);CHKERRQ(ierr);}
if (!redundantPoints) {ierr = PetscFree3(globalPoints,counts,displs);CHKERRQ(ierr);}
ierr = PetscLayoutDestroy(&layout);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
