/*@
ISSortPermutation - calculate the permutation of the indices into a nondecreasing order.
Not collective.
Input arguments:
+ f - IS to sort
- always - build the permutation even when f's indices are nondecreasin.
Output argument:
. h - permutation or NULL, if f is nondecreasing and always == PETSC_TRUE.
Note: Indices in f are unchanged. f[h[i]] is the i-th smallest f index.
If always == PETSC_FALSE, an extra check is peformed to see whether
the f indices are nondecreasing. h is built on PETSC_COMM_SELF, since
the permutation has a local meaning only.
Level: advanced
.seealso ISLocalToGlobalMapping, ISSort(), PetscIntSortWithPermutation()
@*/
PetscErrorCode ISSortPermutation(IS f,PetscBool always,IS *h)
{
PetscErrorCode ierr;
const PetscInt *findices;
PetscInt fsize,*hindices,i;
PetscBool isincreasing;
PetscFunctionBegin;
PetscValidHeaderSpecific(f,IS_CLASSID,1);
PetscValidPointer(h,3);
ierr = ISGetLocalSize(f,&fsize);CHKERRQ(ierr);
ierr = ISGetIndices(f,&findices);CHKERRQ(ierr);
*h = NULL;
if (!always) {
isincreasing = PETSC_TRUE;
for (i = 1; i < fsize; ++i) {
if (findices[i] <= findices[i-1]) {
isincreasing = PETSC_FALSE;
break;
}
}
if (isincreasing) {
ierr = ISRestoreIndices(f,&findices);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
}
ierr = PetscMalloc1(fsize,&hindices);CHKERRQ(ierr);
for (i = 0; i < fsize; ++i) hindices[i] = i;
ierr = PetscSortIntWithPermutation(fsize,findices,hindices);CHKERRQ(ierr);
ierr = ISRestoreIndices(f,&findices);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,fsize,hindices,PETSC_OWN_POINTER,h);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscInt i;
PetscInt x[] = {39, 9, 39, 39, 29},index[5];
PetscInt x2[] = {39, 9, 19, 39, 29, 39, 29, 39},index2[8];
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
printf("1st test\n");
for (i=0; i<5; i++) index[i] = i;
ierr = PetscSortIntWithPermutation(5, x, index);
for (i=0; i<5; i++) printf(" %d %d %d\n",x[i], index[i],x[index[i]]);
printf("\n2nd test\n");
for (i=0; i<8; i++) index2[i] = i;
ierr = PetscSortIntWithPermutation(8, x2, index2);
for (i=0; i<8; i++) printf(" %d %d %d\n",x2[i], index2[i],x2[index2[i]]);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
PetscMPIInt rank,size,*toranks,*fromranks,nto,nfrom;
PetscInt i,n;
PetscBool verbose,build_twosided_f;
Unit *todata,*fromdata;
MPI_Datatype dtype;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
verbose = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-verbose",&verbose,NULL);CHKERRQ(ierr);
build_twosided_f = PETSC_FALSE;
ierr = PetscOptionsGetBool(NULL,"-build_twosided_f",&build_twosided_f,NULL);CHKERRQ(ierr);
for (i=1,nto=0; i<size; i*=2) nto++;
ierr = PetscMalloc2(nto,&todata,nto,&toranks);CHKERRQ(ierr);
for (n=0,i=1; i<size; n++,i*=2) {
toranks[n] = (rank+i) % size;
todata[n].rank = (rank+i) % size;
todata[n].value = (PetscScalar)rank;
todata[n].ok[0] = 'o';
todata[n].ok[1] = 'k';
todata[n].ok[2] = 0;
}
if (verbose) {
for (i=0; i<nto; i++) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] TO %d: {%D, %g, \"%s\"}\n",rank,toranks[i],todata[i].rank,(double)PetscRealPart(todata[i].value),todata[i].ok);CHKERRQ(ierr);
}
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
}
ierr = MakeDatatype(&dtype);CHKERRQ(ierr);
if (build_twosided_f) {
struct FCtx fctx;
PetscMPIInt *todummy,*fromdummy;
fctx.rank = rank;
fctx.nto = nto;
fctx.toranks = toranks;
fctx.todata = todata;
ierr = PetscSegBufferCreate(sizeof(Unit),1,&fctx.seg);CHKERRQ(ierr);
ierr = PetscMalloc1(nto,&todummy);CHKERRQ(ierr);
for (i=0; i<nto; i++) todummy[i] = rank;
ierr = PetscCommBuildTwoSidedF(PETSC_COMM_WORLD,1,MPI_INT,nto,toranks,todummy,&nfrom,&fromranks,&fromdummy,2,FSend,FRecv,&fctx);CHKERRQ(ierr);
ierr = PetscFree(todummy);CHKERRQ(ierr);
ierr = PetscFree(fromdummy);CHKERRQ(ierr);
ierr = PetscSegBufferExtractAlloc(fctx.seg,&fromdata);CHKERRQ(ierr);
ierr = PetscSegBufferDestroy(&fctx.seg);CHKERRQ(ierr);
} else {
ierr = PetscCommBuildTwoSided(PETSC_COMM_WORLD,1,dtype,nto,toranks,todata,&nfrom,&fromranks,&fromdata);CHKERRQ(ierr);
}
ierr = MPI_Type_free(&dtype);CHKERRQ(ierr);
if (verbose) {
PetscInt *iranks,*iperm;
ierr = PetscMalloc2(nfrom,&iranks,nfrom,&iperm);CHKERRQ(ierr);
for (i=0; i<nfrom; i++) {
iranks[i] = fromranks[i];
iperm[i] = i;
}
/* Receive ordering is non-deterministic in general, so sort to make verbose output deterministic. */
ierr = PetscSortIntWithPermutation(nfrom,iranks,iperm);CHKERRQ(ierr);
for (i=0; i<nfrom; i++) {
PetscInt ip = iperm[i];
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] FROM %d: {%D, %g, \"%s\"}\n",rank,fromranks[ip],fromdata[ip].rank,(double)PetscRealPart(fromdata[ip].value),fromdata[ip].ok);CHKERRQ(ierr);
}
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
ierr = PetscFree2(iranks,iperm);CHKERRQ(ierr);
}
if (nto != nfrom) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_PLIB,"[%d] From ranks %d does not match To ranks %d",rank,nto,nfrom);
for (i=1; i<size; i*=2) {
PetscMPIInt expected_rank = (rank-i+size)%size;
PetscBool flg;
for (n=0; n<nfrom; n++) {
if (expected_rank == fromranks[n]) goto found;
}
SETERRQ2(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"[%d] Could not find expected from rank %d",rank,expected_rank);
found:
if (PetscRealPart(fromdata[n].value) != expected_rank) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_PLIB,"[%d] Got data %g from rank %d",rank,(double)PetscRealPart(fromdata[n].value),expected_rank);
ierr = PetscStrcmp(fromdata[n].ok,"ok",&flg);CHKERRQ(ierr);
if (!flg) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_PLIB,"[%d] Got string %s from rank %d",rank,fromdata[n].ok,expected_rank);
}
ierr = PetscFree2(todata,toranks);CHKERRQ(ierr);
ierr = PetscFree(fromdata);CHKERRQ(ierr);
ierr = PetscFree(fromranks);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
/*@C
AOCreateMapping - Creates a basic application mapping using two integer arrays.
Input Parameters:
+ comm - MPI communicator that is to share AO
. napp - size of integer arrays
. myapp - integer array that defines an ordering
- mypetsc - integer array that defines another ordering (may be NULL to indicate the identity ordering)
Output Parameter:
. aoout - the new application mapping
Options Database Key:
. -ao_view : call AOView() at the conclusion of AOCreateMapping()
Level: beginner
Notes: the arrays myapp and mypetsc need NOT contain the all the integers 0 to napp-1, that is there CAN be "holes" in the indices.
Use AOCreateBasic() or AOCreateBasicIS() if they do not have holes for better performance.
.keywords: AO, create
.seealso: AOCreateBasic(), AOCreateBasic(), AOCreateMappingIS(), AODestroy()
@*/
PetscErrorCode AOCreateMapping(MPI_Comm comm,PetscInt napp,const PetscInt myapp[],const PetscInt mypetsc[],AO *aoout)
{
AO ao;
AO_Mapping *aomap;
PetscInt *allpetsc, *allapp;
PetscInt *petscPerm, *appPerm;
PetscInt *petsc;
PetscMPIInt size, rank,*lens, *disp,nnapp;
PetscInt N, start;
PetscInt i;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidPointer(aoout,5);
*aoout = 0;
ierr = AOInitializePackage();CHKERRQ(ierr);
ierr = PetscHeaderCreate(ao, AO_CLASSID, "AO", "Application Ordering", "AO", comm, AODestroy, AOView);CHKERRQ(ierr);
ierr = PetscNewLog(ao,&aomap);CHKERRQ(ierr);
ierr = PetscMemcpy(ao->ops, &AOps, sizeof(AOps));CHKERRQ(ierr);
ao->data = (void*) aomap;
/* transmit all lengths to all processors */
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = PetscMalloc2(size, &lens,size,&disp);CHKERRQ(ierr);
nnapp = napp;
ierr = MPI_Allgather(&nnapp, 1, MPI_INT, lens, 1, MPI_INT, comm);CHKERRQ(ierr);
N = 0;
for (i = 0; i < size; i++) {
disp[i] = N;
N += lens[i];
}
aomap->N = N;
ao->N = N;
ao->n = N;
/* If mypetsc is 0 then use "natural" numbering */
if (!mypetsc) {
start = disp[rank];
ierr = PetscMalloc1(napp+1, &petsc);CHKERRQ(ierr);
for (i = 0; i < napp; i++) petsc[i] = start + i;
} else {
petsc = (PetscInt*)mypetsc;
}
/* get all indices on all processors */
ierr = PetscMalloc4(N, &allapp,N,&appPerm,N,&allpetsc,N,&petscPerm);CHKERRQ(ierr);
ierr = MPI_Allgatherv((void*)myapp, napp, MPIU_INT, allapp, lens, disp, MPIU_INT, comm);CHKERRQ(ierr);
ierr = MPI_Allgatherv((void*)petsc, napp, MPIU_INT, allpetsc, lens, disp, MPIU_INT, comm);CHKERRQ(ierr);
ierr = PetscFree2(lens,disp);CHKERRQ(ierr);
/* generate a list of application and PETSc node numbers */
ierr = PetscMalloc4(N, &aomap->app,N,&aomap->appPerm,N,&aomap->petsc,N,&aomap->petscPerm);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)ao, 4*N * sizeof(PetscInt));CHKERRQ(ierr);
for (i = 0; i < N; i++) {
appPerm[i] = i;
petscPerm[i] = i;
}
ierr = PetscSortIntWithPermutation(N, allpetsc, petscPerm);CHKERRQ(ierr);
ierr = PetscSortIntWithPermutation(N, allapp, appPerm);CHKERRQ(ierr);
/* Form sorted arrays of indices */
for (i = 0; i < N; i++) {
aomap->app[i] = allapp[appPerm[i]];
aomap->petsc[i] = allpetsc[petscPerm[i]];
}
/* Invert petscPerm[] into aomap->petscPerm[] */
for (i = 0; i < N; i++) aomap->petscPerm[petscPerm[i]] = i;
/* Form map between aomap->app[] and aomap->petsc[] */
for (i = 0; i < N; i++) aomap->appPerm[i] = aomap->petscPerm[appPerm[i]];
/* Invert appPerm[] into allapp[] */
for (i = 0; i < N; i++) allapp[appPerm[i]] = i;
/* Form map between aomap->petsc[] and aomap->app[] */
for (i = 0; i < N; i++) aomap->petscPerm[i] = allapp[petscPerm[i]];
#if defined(PETSC_USE_DEBUG)
/* Check that the permutations are complementary */
for (i = 0; i < N; i++) {
if (i != aomap->appPerm[aomap->petscPerm[i]]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB, "Invalid ordering");
}
#endif
/* Cleanup */
if (!mypetsc) {
ierr = PetscFree(petsc);CHKERRQ(ierr);
}
ierr = PetscFree4(allapp,appPerm,allpetsc,petscPerm);CHKERRQ(ierr);
ierr = AOViewFromOptions(ao,NULL,"-ao_view");CHKERRQ(ierr);
*aoout = ao;
PetscFunctionReturn(0);
}
