PetscErrorCode MatPartitioningHierarchical_ReassembleFineparts(Mat adj, IS fineparts, ISLocalToGlobalMapping mapping, IS *sfineparts)
{
PetscInt *local_indices, *global_indices,*owners,*sfineparts_indices,localsize,i;
const PetscInt *ranges,*fineparts_indices;
PetscMPIInt rank;
MPI_Comm comm;
PetscLayout rmap;
PetscSFNode *remote;
PetscSF sf;
PetscErrorCode ierr;
PetscFunctionBegin;
/*get communicator */
ierr = PetscObjectGetComm((PetscObject)adj,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MatGetLayouts(adj,&rmap,PETSC_NULL);CHKERRQ(ierr);
ierr = ISGetLocalSize(fineparts,&localsize);CHKERRQ(ierr);
ierr = PetscCalloc2(localsize,&global_indices,localsize,&local_indices);CHKERRQ(ierr);
for(i=0; i<localsize; i++){
local_indices[i] = i;
}
/*global indices */
ierr = ISLocalToGlobalMappingApply(mapping,localsize,local_indices,global_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(localsize,&owners);CHKERRQ(ierr);
/*find owners for global indices */
for(i=0; i<localsize; i++){
ierr = PetscLayoutFindOwner(rmap,global_indices[i],&owners[i]);CHKERRQ(ierr);
}
/*ranges */
ierr = PetscLayoutGetRanges(rmap,&ranges);CHKERRQ(ierr);
ierr = PetscCalloc1(ranges[rank+1]-ranges[rank],&sfineparts_indices);CHKERRQ(ierr);
ierr = ISGetIndices(fineparts,&fineparts_indices);CHKERRQ(ierr);
/*create a SF to exchange data */
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscCalloc1(localsize,&remote);CHKERRQ(ierr);
for(i=0; i<localsize; i++){
remote[i].rank = owners[i];
remote[i].index = global_indices[i]-ranges[owners[i]];
}
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
/*not sure how to add prefix to sf*/
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,localsize,localsize,PETSC_NULL,PETSC_OWN_POINTER,remote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sf,MPIU_INT,fineparts_indices,sfineparts_indices,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sf,MPIU_INT,fineparts_indices,sfineparts_indices,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = ISRestoreIndices(fineparts,&fineparts_indices);CHKERRQ(ierr);
/* comm self */
ierr = ISCreateGeneral(comm,ranges[rank+1]-ranges[rank],sfineparts_indices,PETSC_OWN_POINTER,sfineparts);CHKERRQ(ierr);
ierr = PetscFree2(global_indices,local_indices);CHKERRQ(ierr);
ierr = PetscFree(owners);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatISSetPreallocation_IS(Mat B,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
{
Mat_IS *matis = (Mat_IS*)(B->data);
PetscSF sf;
PetscInt bs,i,nroots,*rootdata,nleaves,*leafdata,nlocalcols;
const PetscInt *gidxs;
PetscErrorCode ierr;
PetscFunctionBegin;
if (!matis->A) {
SETERRQ(PetscObjectComm((PetscObject)B),PETSC_ERR_SUP,"You should first call MatSetLocalToGlobalMapping");
}
ierr = MatGetLocalSize(B,&nroots,NULL);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&nleaves,&nlocalcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(matis->A,&bs);CHKERRQ(ierr);
ierr = PetscCalloc2(nroots,&rootdata,nleaves,&leafdata);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)B),&sf);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,B->rmap,nleaves,NULL,PETSC_COPY_VALUES,gidxs);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingRestoreIndices(matis->mapping,&gidxs);CHKERRQ(ierr);
if (!d_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += d_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += d_nnz[i];
}
if (!o_nnz) {
for (i=0;i<nroots;i++) rootdata[i] += o_nz;
} else {
for (i=0;i<nroots;i++) rootdata[i] += o_nnz[i];
}
ierr = PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves;i++) {
leafdata[i] = PetscMin(leafdata[i],nlocalcols);
}
ierr = MatSeqAIJSetPreallocation(matis->A,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i*bs]/bs;
}
ierr = MatSeqBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
for (i=0;i<nleaves/bs;i++) {
leafdata[i] = leafdata[i]-i;
}
ierr = MatSeqSBAIJSetPreallocation(matis->A,bs,0,leafdata);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree2(rootdata,leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatSetUp_Preallocator(Mat A)
{
Mat_Preallocator *p = (Mat_Preallocator *) A->data;
PetscInt m, bs;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscLayoutSetUp(A->rmap);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(A->cmap);CHKERRQ(ierr);
ierr = MatGetLocalSize(A, &m, NULL);CHKERRQ(ierr);
ierr = PetscHSetIJCreate(&p->ht);CHKERRQ(ierr);
ierr = MatGetBlockSize(A, &bs);CHKERRQ(ierr);
ierr = MatStashCreate_Private(PetscObjectComm((PetscObject) A), bs, &A->stash);CHKERRQ(ierr);
ierr = PetscCalloc2(m, &p->dnz, m, &p->onz);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*@C
PetscSFCreateEmbeddedSF - removes edges from all but the selected roots, does not remap indices
Collective
Input Arguments:
+ sf - original star forest
. nroots - number of roots to select on this process
- selected - selected roots on this process
Output Arguments:
. newsf - new star forest
Level: advanced
Note:
To use the new PetscSF, it may be necessary to know the indices of the leaves that are still participating. This can
be done by calling PetscSFGetGraph().
.seealso: PetscSFSetGraph(), PetscSFGetGraph()
@*/
PetscErrorCode PetscSFCreateEmbeddedSF(PetscSF sf,PetscInt nroots,const PetscInt *selected,PetscSF *newsf)
{
PetscInt *rootdata, *leafdata, *ilocal;
PetscSFNode *iremote;
PetscInt leafsize = 0, nleaves = 0, n, i;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(sf,PETSCSF_CLASSID,1);
if (nroots) PetscValidPointer(selected,3);
PetscValidPointer(newsf,4);
if (sf->mine) for (i = 0; i < sf->nleaves; ++i) {leafsize = PetscMax(leafsize, sf->mine[i]+1);}
else leafsize = sf->nleaves;
ierr = PetscCalloc2(sf->nroots,&rootdata,leafsize,&leafdata);CHKERRQ(ierr);
for (i=0; i<nroots; ++i) rootdata[selected[i]] = 1;
ierr = PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);CHKERRQ(ierr);
for (i = 0; i < leafsize; ++i) nleaves += leafdata[i];
ierr = PetscMalloc1(nleaves,&ilocal);CHKERRQ(ierr);
ierr = PetscMalloc1(nleaves,&iremote);CHKERRQ(ierr);
for (i = 0, n = 0; i < sf->nleaves; ++i) {
const PetscInt lidx = sf->mine ? sf->mine[i] : i;
if (leafdata[lidx]) {
ilocal[n] = lidx;
iremote[n].rank = sf->remote[i].rank;
iremote[n].index = sf->remote[i].index;
++n;
}
}
if (n != nleaves) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "There is a size mismatch in the SF embedding, %d != %d", n, nleaves);
ierr = PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,newsf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(*newsf,sf->nroots,nleaves,ilocal,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscFree2(rootdata,leafdata);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/*
* Increase overlap for the sub-matrix across sub communicator
* sub-matrix could be a graph or numerical matrix
* */
PetscErrorCode MatIncreaseOverlapSplit_Single(Mat mat,IS *is,PetscInt ov)
{
PetscInt i,nindx,*indices_sc,*indices_ov,localsize,*localsizes_sc,localsize_tmp;
PetscInt *indices_ov_rd,nroots,nleaves,*localoffsets,*indices_recv,*sources_sc,*sources_sc_rd;
const PetscInt *indices;
PetscMPIInt srank,ssize,issamecomm,k,grank;
IS is_sc,allis_sc,partitioning;
MPI_Comm gcomm,dcomm,scomm;
PetscSF sf;
PetscSFNode *remote;
Mat *smat;
MatPartitioning part;
PetscErrorCode ierr;
PetscFunctionBegin;
/* get a sub communicator before call individual MatIncreaseOverlap
* since the sub communicator may be changed.
* */
ierr = PetscObjectGetComm((PetscObject)(*is),&dcomm);CHKERRQ(ierr);
/*make a copy before the original one is deleted*/
ierr = PetscCommDuplicate(dcomm,&scomm,NULL);CHKERRQ(ierr);
/*get a global communicator, where mat should be a global matrix */
ierr = PetscObjectGetComm((PetscObject)mat,&gcomm);CHKERRQ(ierr);
/*increase overlap on each individual subdomain*/
ierr = (*mat->ops->increaseoverlap)(mat,1,is,ov);CHKERRQ(ierr);
/*compare communicators */
ierr = MPI_Comm_compare(gcomm,scomm,&issamecomm);CHKERRQ(ierr);
/* if the sub-communicator is the same as the global communicator,
* user does not want to use a sub-communicator
* */
if(issamecomm == MPI_IDENT || issamecomm == MPI_CONGRUENT) PetscFunctionReturn(0);
/* if the sub-communicator is petsc_comm_self,
* user also does not care the sub-communicator
* */
ierr = MPI_Comm_compare(scomm,PETSC_COMM_SELF,&issamecomm);CHKERRQ(ierr);
if(issamecomm == MPI_IDENT || issamecomm == MPI_CONGRUENT){PetscFunctionReturn(0);}
/*local rank, size in a sub-communicator */
ierr = MPI_Comm_rank(scomm,&srank);CHKERRQ(ierr);
ierr = MPI_Comm_size(scomm,&ssize);CHKERRQ(ierr);
ierr = MPI_Comm_rank(gcomm,&grank);CHKERRQ(ierr);
/*create a new IS based on sub-communicator
* since the old IS is often based on petsc_comm_self
* */
ierr = ISGetLocalSize(*is,&nindx);CHKERRQ(ierr);
ierr = PetscCalloc1(nindx,&indices_sc);CHKERRQ(ierr);
ierr = ISGetIndices(*is,&indices);CHKERRQ(ierr);
ierr = PetscMemcpy(indices_sc,indices,sizeof(PetscInt)*nindx);CHKERRQ(ierr);
ierr = ISRestoreIndices(*is,&indices);CHKERRQ(ierr);
/*we do not need any more*/
ierr = ISDestroy(is);CHKERRQ(ierr);
/*create a index set based on the sub communicator */
ierr = ISCreateGeneral(scomm,nindx,indices_sc,PETSC_OWN_POINTER,&is_sc);CHKERRQ(ierr);
/*gather all indices within the sub communicator*/
ierr = ISAllGather(is_sc,&allis_sc);CHKERRQ(ierr);
ierr = ISDestroy(&is_sc);CHKERRQ(ierr);
/* gather local sizes */
ierr = PetscMalloc1(ssize,&localsizes_sc);CHKERRQ(ierr);
/*get individual local sizes for all index sets*/
ierr = MPI_Gather(&nindx,1,MPIU_INT,localsizes_sc,1,MPIU_INT,0,scomm);CHKERRQ(ierr);
/*only root does these computations */
if(!srank){
/*get local size for the big index set*/
ierr = ISGetLocalSize(allis_sc,&localsize);CHKERRQ(ierr);
ierr = PetscCalloc2(localsize,&indices_ov,localsize,&sources_sc);CHKERRQ(ierr);
ierr = PetscCalloc2(localsize,&indices_ov_rd,localsize,&sources_sc_rd);CHKERRQ(ierr);
ierr = ISGetIndices(allis_sc,&indices);CHKERRQ(ierr);
ierr = PetscMemcpy(indices_ov,indices,sizeof(PetscInt)*localsize);CHKERRQ(ierr);
ierr = ISRestoreIndices(allis_sc,&indices);CHKERRQ(ierr);
/*we do not need it any more */
ierr = ISDestroy(&allis_sc);CHKERRQ(ierr);
/*assign corresponding sources */
localsize_tmp = 0;
for(k=0; k<ssize; k++){
for(i=0; i<localsizes_sc[k]; i++){
sources_sc[localsize_tmp++] = k;
}
}
/*record where indices come from */
ierr = PetscSortIntWithArray(localsize,indices_ov,sources_sc);CHKERRQ(ierr);
/*count local sizes for reduced indices */
ierr = PetscMemzero(localsizes_sc,sizeof(PetscInt)*ssize);CHKERRQ(ierr);
/*initialize the first entity*/
if(localsize){
indices_ov_rd[0] = indices_ov[0];
sources_sc_rd[0] = sources_sc[0];
localsizes_sc[sources_sc[0]]++;
}
localsize_tmp = 1;
/*remove duplicate integers */
for(i=1; i<localsize; i++){
if(indices_ov[i] != indices_ov[i-1]){
indices_ov_rd[localsize_tmp] = indices_ov[i];
sources_sc_rd[localsize_tmp++] = sources_sc[i];
localsizes_sc[sources_sc[i]]++;
}
}
ierr = PetscFree2(indices_ov,sources_sc);CHKERRQ(ierr);
ierr = PetscCalloc1(ssize+1,&localoffsets);CHKERRQ(ierr);
for(k=0; k<ssize; k++){
localoffsets[k+1] = localoffsets[k] + localsizes_sc[k];
}
/*construct a star forest to send data back */
nleaves = localoffsets[ssize];
ierr = PetscMemzero(localoffsets,(ssize+1)*sizeof(PetscInt));CHKERRQ(ierr);
nroots = localsizes_sc[srank];
ierr = PetscCalloc1(nleaves,&remote);CHKERRQ(ierr);
for(i=0; i<nleaves; i++){
remote[i].rank = sources_sc_rd[i];
remote[i].index = localoffsets[sources_sc_rd[i]]++;
}
ierr = PetscFree(localoffsets);CHKERRQ(ierr);
}else{
ierr = ISDestroy(&allis_sc);CHKERRQ(ierr);
/*Allocate a 'zero' pointer */
ierr = PetscCalloc1(0,&remote);CHKERRQ(ierr);
nleaves = 0;
indices_ov_rd = 0;
sources_sc_rd = 0;
}
/*scatter sizes to everybody */
ierr = MPI_Scatter(localsizes_sc,1, MPIU_INT,&nroots,1, MPIU_INT,0,scomm);CHKERRQ(ierr);
/*free memory */
ierr = PetscFree(localsizes_sc);CHKERRQ(ierr);
ierr = PetscCalloc1(nroots,&indices_recv);CHKERRQ(ierr);
/*ierr = MPI_Comm_dup(scomm,&dcomm);CHKERRQ(ierr);*/
/*set data back to every body */
ierr = PetscSFCreate(scomm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,remote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sf,MPIU_INT,indices_ov_rd,indices_recv,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sf,MPIU_INT,indices_ov_rd,indices_recv,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
/* free memory */
ierr = PetscFree2(indices_ov_rd,sources_sc_rd);CHKERRQ(ierr);
/*create a index set*/
ierr = ISCreateGeneral(scomm,nroots,indices_recv,PETSC_OWN_POINTER,&is_sc);CHKERRQ(ierr);
/*construct a parallel submatrix */
ierr = MatGetSubMatricesMPI(mat,1,&is_sc,&is_sc,MAT_INITIAL_MATRIX,&smat);CHKERRQ(ierr);
/* we do not need them any more */
ierr = ISDestroy(&allis_sc);CHKERRQ(ierr);
/*create a partitioner to repartition the sub-matrix*/
ierr = MatPartitioningCreate(scomm,&part);CHKERRQ(ierr);
ierr = MatPartitioningSetAdjacency(part,smat[0]);CHKERRQ(ierr);
#if PETSC_HAVE_PARMETIS
/* if there exists a ParMETIS installation, we try to use ParMETIS
* because a repartition routine possibly work better
* */
ierr = MatPartitioningSetType(part,MATPARTITIONINGPARMETIS);CHKERRQ(ierr);
/*try to use reparition function, instead of partition function */
ierr = MatPartitioningParmetisSetRepartition(part);CHKERRQ(ierr);
#else
/*we at least provide a default partitioner to rebalance the computation */
ierr = MatPartitioningSetType(part,MATPARTITIONINGAVERAGE);CHKERRQ(ierr);
#endif
/*user can pick up any partitioner by using an option*/
ierr = MatPartitioningSetFromOptions(part);CHKERRQ(ierr);
/* apply partition */
ierr = MatPartitioningApply(part,&partitioning);CHKERRQ(ierr);
ierr = MatPartitioningDestroy(&part);CHKERRQ(ierr);
ierr = MatDestroy(&(smat[0]));CHKERRQ(ierr);
ierr = PetscFree(smat);CHKERRQ(ierr);
/* get local rows including overlap */
ierr = ISBuildTwoSided(partitioning,is_sc,is);CHKERRQ(ierr);
/* destroy */
ierr = ISDestroy(&is_sc);CHKERRQ(ierr);
ierr = ISDestroy(&partitioning);CHKERRQ(ierr);
ierr = PetscCommDestroy(&scomm);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PETSC_EXTERN PetscErrorCode MatISSetMPIXAIJPreallocation_Private(Mat A, Mat B, PetscBool maxreduce)
{
Mat_IS *matis = (Mat_IS*)(A->data);
PetscInt *my_dnz,*my_onz,*dnz,*onz,*mat_ranges,*row_ownership;
const PetscInt *global_indices_r,*global_indices_c;
PetscInt i,j,bs,rows,cols;
PetscInt lrows,lcols;
PetscInt local_rows,local_cols;
PetscMPIInt nsubdomains;
PetscBool isdense,issbaij;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&nsubdomains);CHKERRQ(ierr);
ierr = MatGetSize(A,&rows,&cols);CHKERRQ(ierr);
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
ierr = MatGetSize(matis->A,&local_rows,&local_cols);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQDENSE,&isdense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)matis->A,MATSEQSBAIJ,&issbaij);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr);
if (A->rmap->mapping != A->cmap->mapping) {
ierr = ISLocalToGlobalMappingGetIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr);
} else {
global_indices_c = global_indices_r;
}
if (issbaij) {
ierr = MatGetRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
/*
An SF reduce is needed to sum up properly on shared rows.
Note that generally preallocation is not exact, since it overestimates nonzeros
*/
if (!matis->sf) { /* setup SF if not yet created and allocate rootdata and leafdata */
ierr = MatISComputeSF_Private(A);CHKERRQ(ierr);
}
ierr = MatGetLocalSize(A,&lrows,&lcols);CHKERRQ(ierr);
ierr = MatPreallocateInitialize(PetscObjectComm((PetscObject)A),lrows,lcols,dnz,onz);CHKERRQ(ierr);
/* All processes need to compute entire row ownership */
ierr = PetscMalloc1(rows,&row_ownership);CHKERRQ(ierr);
ierr = MatGetOwnershipRanges(A,(const PetscInt**)&mat_ranges);CHKERRQ(ierr);
for (i=0;i<nsubdomains;i++) {
for (j=mat_ranges[i];j<mat_ranges[i+1];j++) {
row_ownership[j] = i;
}
}
/*
my_dnz and my_onz contains exact contribution to preallocation from each local mat
then, they will be summed up properly. This way, preallocation is always sufficient
*/
ierr = PetscCalloc2(local_rows,&my_dnz,local_rows,&my_onz);CHKERRQ(ierr);
/* preallocation as a MATAIJ */
if (isdense) { /* special case for dense local matrices */
for (i=0;i<local_rows;i++) {
PetscInt index_row = global_indices_r[i];
for (j=i;j<local_rows;j++) {
PetscInt owner = row_ownership[index_row];
PetscInt index_col = global_indices_c[j];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1;
} else { /* offdiag block */
my_onz[i] += 1;
}
/* same as before, interchanging rows and cols */
if (i != j) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[j] += 1;
} else {
my_onz[j] += 1;
}
}
}
}
} else { /* TODO: this could be optimized using MatGetRowIJ */
for (i=0;i<local_rows;i++) {
const PetscInt *cols;
PetscInt ncols,index_row = global_indices_r[i];
ierr = MatGetRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
PetscInt owner = row_ownership[index_row];
PetscInt index_col = global_indices_c[cols[j]];
if (index_col > mat_ranges[owner]-1 && index_col < mat_ranges[owner+1] ) { /* diag block */
my_dnz[i] += 1;
} else { /* offdiag block */
my_onz[i] += 1;
}
/* same as before, interchanging rows and cols */
if (issbaij && index_col != index_row) {
owner = row_ownership[index_col];
if (index_row > mat_ranges[owner]-1 && index_row < mat_ranges[owner+1] ) {
my_dnz[cols[j]] += 1;
} else {
my_onz[cols[j]] += 1;
}
}
}
ierr = MatRestoreRow(matis->A,i,&ncols,&cols,NULL);CHKERRQ(ierr);
}
}
ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_r);CHKERRQ(ierr);
if (global_indices_c != global_indices_r) {
ierr = ISLocalToGlobalMappingRestoreIndices(A->rmap->mapping,&global_indices_c);CHKERRQ(ierr);
}
ierr = PetscFree(row_ownership);CHKERRQ(ierr);
/* Reduce my_dnz and my_onz */
if (maxreduce) {
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_MAX);CHKERRQ(ierr);
} else {
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_dnz,dnz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(matis->sf,MPIU_INT,my_onz,onz,MPI_SUM);CHKERRQ(ierr);
}
ierr = PetscFree2(my_dnz,my_onz);CHKERRQ(ierr);
/* Resize preallocation if overestimated */
for (i=0;i<lrows;i++) {
dnz[i] = PetscMin(dnz[i],lcols);
onz[i] = PetscMin(onz[i],cols-lcols);
}
/* set preallocation */
ierr = MatMPIAIJSetPreallocation(B,0,dnz,0,onz);CHKERRQ(ierr);
for (i=0;i<lrows/bs;i++) {
dnz[i] = dnz[i*bs]/bs;
onz[i] = onz[i*bs]/bs;
}
ierr = MatMPIBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(B,bs,0,dnz,0,onz);CHKERRQ(ierr);
ierr = MatPreallocateFinalize(dnz,onz);CHKERRQ(ierr);
if (issbaij) {
ierr = MatRestoreRowUpperTriangular(matis->A);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscErrorCode ierr;
DS ds;
SlepcSC sc;
PetscReal *T,*s,re,im;
PetscScalar *eigr,*eigi;
PetscInt i,n=10,l=2,k=5,ld;
PetscViewer viewer;
PetscBool verbose;
SlepcInitialize(&argc,&argv,(char*)0,help);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Solve a Dense System of type GHIEP with compact storage - dimension %D.\n",n);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-l",&l,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-k",&k,NULL);CHKERRQ(ierr);
if (l>n || k>n || l>k) SETERRQ(PETSC_COMM_WORLD,1,"Wrong value of dimensions");
ierr = PetscOptionsHasName(NULL,"-verbose",&verbose);CHKERRQ(ierr);
/* Create DS object */
ierr = DSCreate(PETSC_COMM_WORLD,&ds);CHKERRQ(ierr);
ierr = DSSetType(ds,DSGHIEP);CHKERRQ(ierr);
ierr = DSSetFromOptions(ds);CHKERRQ(ierr);
ld = n+2; /* test leading dimension larger than n */
ierr = DSAllocate(ds,ld);CHKERRQ(ierr);
ierr = DSSetDimensions(ds,n,0,l,k);CHKERRQ(ierr);
ierr = DSSetCompact(ds,PETSC_TRUE);CHKERRQ(ierr);
/* Set up viewer */
ierr = PetscViewerASCIIGetStdout(PETSC_COMM_WORLD,&viewer);CHKERRQ(ierr);
ierr = PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_INFO_DETAIL);CHKERRQ(ierr);
ierr = DSView(ds,viewer);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(viewer);CHKERRQ(ierr);
if (verbose) {
ierr = PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);CHKERRQ(ierr);
}
/* Fill arrow-tridiagonal matrix */
ierr = DSGetArrayReal(ds,DS_MAT_T,&T);CHKERRQ(ierr);
ierr = DSGetArrayReal(ds,DS_MAT_D,&s);CHKERRQ(ierr);
for (i=0;i<n;i++) T[i] = (PetscReal)(i+1);
for (i=k;i<n-1;i++) T[i+ld] = 1.0;
for (i=l;i<k;i++) T[i+2*ld] = 1.0;
T[2*ld+l+1] = -7; T[ld+k+1] = -7;
/* Signature matrix */
for (i=0;i<n;i++) s[i] = 1.0;
s[l+1] = -1.0;
s[k+1] = -1.0;
ierr = DSRestoreArrayReal(ds,DS_MAT_T,&T);CHKERRQ(ierr);
ierr = DSRestoreArrayReal(ds,DS_MAT_D,&s);CHKERRQ(ierr);
if (l==0 && k==0) {
ierr = DSSetState(ds,DS_STATE_INTERMEDIATE);CHKERRQ(ierr);
} else {
ierr = DSSetState(ds,DS_STATE_RAW);CHKERRQ(ierr);
}
if (verbose) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Initial - - - - - - - - -\n");CHKERRQ(ierr);
ierr = DSView(ds,viewer);CHKERRQ(ierr);
}
/* Solve */
ierr = PetscCalloc2(n,&eigr,n,&eigi);CHKERRQ(ierr);
ierr = DSGetSlepcSC(ds,&sc);CHKERRQ(ierr);
sc->comparison = SlepcCompareLargestMagnitude;
sc->comparisonctx = NULL;
sc->map = NULL;
sc->mapobj = NULL;
ierr = DSSolve(ds,eigr,eigi);CHKERRQ(ierr);
ierr = DSSort(ds,eigr,eigi,NULL,NULL,NULL);CHKERRQ(ierr);
if (verbose) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"After solve - - - - - - - - -\n");CHKERRQ(ierr);
ierr = DSView(ds,viewer);CHKERRQ(ierr);
}
/* Print eigenvalues */
ierr = PetscPrintf(PETSC_COMM_WORLD,"Computed eigenvalues =\n",n);CHKERRQ(ierr);
for (i=0;i<n;i++) {
#if defined(PETSC_USE_COMPLEX)
re = PetscRealPart(eigr[i]);
im = PetscImaginaryPart(eigr[i]);
#else
re = eigr[i];
im = eigi[i];
#endif
if (PetscAbs(im)<1e-10) {
ierr = PetscViewerASCIIPrintf(viewer," %.5f\n",(double)re);CHKERRQ(ierr);
} else {
ierr = PetscViewerASCIIPrintf(viewer," %.5f%+.5fi\n",(double)re,(double)im);CHKERRQ(ierr);
}
}
ierr = PetscFree2(eigr,eigi);CHKERRQ(ierr);
ierr = DSDestroy(&ds);CHKERRQ(ierr);
ierr = SlepcFinalize();
return 0;
}
/*@
ISBuildTwoSided - Takes an IS that describes where we will go. Generates an IS that contains new numbers from remote or local
on the IS.
Collective on IS
Input Parameters
. to - an IS describes where we will go. Negative target rank will be ignored
. toindx - an IS describes what indices should send. NULL means sending natural numbering
Output Parameter:
. rows - contains new numbers from remote or local
Level: advanced
.seealso: MatPartitioningCreate(), ISPartitioningToNumbering(), ISPartitioningCount()
@*/
PetscErrorCode ISBuildTwoSided(IS ito,IS toindx, IS *rows)
{
const PetscInt *ito_indices,*toindx_indices;
PetscInt *send_indices,rstart,*recv_indices,nrecvs,nsends;
PetscInt *tosizes,*fromsizes,i,j,*tosizes_tmp,*tooffsets_tmp,ito_ln;
PetscMPIInt *toranks,*fromranks,size,target_rank,*fromperm_newtoold,nto,nfrom;
PetscLayout isrmap;
MPI_Comm comm;
PetscSF sf;
PetscSFNode *iremote;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)ito,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = ISGetLocalSize(ito,&ito_ln);CHKERRQ(ierr);
/* why we do not have ISGetLayout? */
isrmap = ito->map;
ierr = PetscLayoutGetRange(isrmap,&rstart,NULL);CHKERRQ(ierr);
ierr = ISGetIndices(ito,&ito_indices);CHKERRQ(ierr);
ierr = PetscCalloc2(size,&tosizes_tmp,size+1,&tooffsets_tmp);CHKERRQ(ierr);
for(i=0; i<ito_ln; i++){
if(ito_indices[i]<0) continue;
#if defined(PETSC_USE_DEBUG)
if(ito_indices[i]>=size) SETERRQ2(comm,PETSC_ERR_ARG_OUTOFRANGE,"target rank %d is larger than communicator size %d ",ito_indices[i],size);
#endif
tosizes_tmp[ito_indices[i]]++;
}
nto = 0;
for(i=0; i<size; i++){
tooffsets_tmp[i+1] = tooffsets_tmp[i]+tosizes_tmp[i];
if(tosizes_tmp[i]>0) nto++;
}
ierr = PetscCalloc2(nto,&toranks,2*nto,&tosizes);CHKERRQ(ierr);
nto = 0;
for(i=0; i<size; i++){
if(tosizes_tmp[i]>0){
toranks[nto] = i;
tosizes[2*nto] = tosizes_tmp[i];/* size */
tosizes[2*nto+1] = tooffsets_tmp[i];/* offset */
nto++;
}
}
nsends = tooffsets_tmp[size];
ierr = PetscCalloc1(nsends,&send_indices);CHKERRQ(ierr);
if(toindx){
ierr = ISGetIndices(toindx,&toindx_indices);CHKERRQ(ierr);
}
for(i=0; i<ito_ln; i++){
if(ito_indices[i]<0) continue;
target_rank = ito_indices[i];
send_indices[tooffsets_tmp[target_rank]] = toindx? toindx_indices[i]:(i+rstart);
tooffsets_tmp[target_rank]++;
}
if(toindx){
ierr = ISRestoreIndices(toindx,&toindx_indices);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(ito,&ito_indices);CHKERRQ(ierr);
ierr = PetscFree2(tosizes_tmp,tooffsets_tmp);CHKERRQ(ierr);
ierr = PetscCommBuildTwoSided(comm,2,MPIU_INT,nto,toranks,tosizes,&nfrom,&fromranks,&fromsizes);CHKERRQ(ierr);
ierr = PetscFree2(toranks,tosizes);CHKERRQ(ierr);
ierr = PetscCalloc1(nfrom,&fromperm_newtoold);CHKERRQ(ierr);
for(i=0; i<nfrom; i++){
fromperm_newtoold[i] = i;
}
ierr = PetscSortMPIIntWithArray(nfrom,fromranks,fromperm_newtoold);CHKERRQ(ierr);
nrecvs = 0;
for(i=0; i<nfrom; i++){
nrecvs += fromsizes[i*2];
}
ierr = PetscCalloc1(nrecvs,&recv_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(nrecvs,&iremote);CHKERRQ(ierr);
nrecvs = 0;
for(i=0; i<nfrom; i++){
for(j=0; j<fromsizes[2*fromperm_newtoold[i]]; j++){
iremote[nrecvs].rank = fromranks[i];
iremote[nrecvs++].index = fromsizes[2*fromperm_newtoold[i]+1]+j;
}
}
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nsends,nrecvs,NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
/* how to put a prefix ? */
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,send_indices,recv_indices);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,send_indices,recv_indices);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree(fromranks);CHKERRQ(ierr);
ierr = PetscFree(fromsizes);CHKERRQ(ierr);
ierr = PetscFree(fromperm_newtoold);CHKERRQ(ierr);
ierr = PetscFree(send_indices);CHKERRQ(ierr);
if(rows){
ierr = PetscSortInt(nrecvs,recv_indices);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, nrecvs,recv_indices,PETSC_OWN_POINTER,rows);CHKERRQ(ierr);
}else{
ierr = PetscFree(recv_indices);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*@
DMPlexCreateCGNS - Create a DMPlex mesh from a CGNS file ID.
Collective on comm
Input Parameters:
+ comm - The MPI communicator
. cgid - The CG id associated with a file and obtained using cg_open
- interpolate - Create faces and edges in the mesh
Output Parameter:
. dm - The DM object representing the mesh
Note: http://www.grc.nasa.gov/WWW/cgns/CGNS_docs_current/index.html
Level: beginner
.keywords: mesh,CGNS
.seealso: DMPlexCreate(), DMPlexCreateExodus()
@*/
PetscErrorCode DMPlexCreateCGNS(MPI_Comm comm, PetscInt cgid, PetscBool interpolate, DM *dm)
{
#if defined(PETSC_HAVE_CGNS)
PetscMPIInt num_proc, rank;
PetscSection coordSection;
Vec coordinates;
PetscScalar *coords;
PetscInt *cellStart, *vertStart;
PetscInt coordSize, v;
PetscErrorCode ierr;
/* Read from file */
char basename[CGIO_MAX_NAME_LENGTH+1];
char buffer[CGIO_MAX_NAME_LENGTH+1];
int dim = 0, physDim = 0, numVertices = 0, numCells = 0;
int nzones = 0;
#endif
PetscFunctionBegin;
#if defined(PETSC_HAVE_CGNS)
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &num_proc);CHKERRQ(ierr);
ierr = DMCreate(comm, dm);CHKERRQ(ierr);
ierr = DMSetType(*dm, DMPLEX);CHKERRQ(ierr);
/* Open CGNS II file and read basic informations on rank 0, then broadcast to all processors */
if (!rank) {
int nbases, z;
ierr = cg_nbases(cgid, &nbases);CHKERRQ(ierr);
if (nbases > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single base, not %d\n",nbases);
ierr = cg_base_read(cgid, 1, basename, &dim, &physDim);CHKERRQ(ierr);
ierr = cg_nzones(cgid, 1, &nzones);CHKERRQ(ierr);
ierr = PetscCalloc2(nzones+1, &cellStart, nzones+1, &vertStart);CHKERRQ(ierr);
for (z = 1; z <= nzones; ++z) {
cgsize_t sizes[3]; /* Number of vertices, number of cells, number of boundary vertices */
ierr = cg_zone_read(cgid, 1, z, buffer, sizes);CHKERRQ(ierr);
numVertices += sizes[0];
numCells += sizes[1];
cellStart[z] += sizes[1] + cellStart[z-1];
vertStart[z] += sizes[0] + vertStart[z-1];
}
for (z = 1; z <= nzones; ++z) {
vertStart[z] += numCells;
}
}
ierr = MPI_Bcast(basename, CGIO_MAX_NAME_LENGTH+1, MPI_CHAR, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&dim, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = MPI_Bcast(&nzones, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) *dm, basename);CHKERRQ(ierr);
ierr = DMSetDimension(*dm, dim);CHKERRQ(ierr);
ierr = DMPlexSetChart(*dm, 0, numCells+numVertices);CHKERRQ(ierr);
/* Read zone information */
if (!rank) {
int z, c, c_loc, v, v_loc;
/* Read the cell set connectivity table and build mesh topology
CGNS standard requires that cells in a zone be numbered sequentially and be pairwise disjoint. */
/* First set sizes */
for (z = 1, c = 0; z <= nzones; ++z) {
ZoneType_t zonetype;
int nsections;
ElementType_t cellType;
cgsize_t start, end;
int nbndry, parentFlag;
PetscInt numCorners;
ierr = cg_zone_type(cgid, 1, z, &zonetype);CHKERRQ(ierr);
if (zonetype == Structured) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Can only handle Unstructured zones for CGNS");
ierr = cg_nsections(cgid, 1, z, &nsections);CHKERRQ(ierr);
if (nsections > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single section, not %d\n",nsections);
ierr = cg_section_read(cgid, 1, z, 1, buffer, &cellType, &start, &end, &nbndry, &parentFlag);CHKERRQ(ierr);
/* This alone is reason enough to bludgeon every single CGNDS developer, this must be what they describe as the "idiocy of crowds" */
if (cellType == MIXED) {
cgsize_t elementDataSize, *elements;
PetscInt off;
ierr = cg_ElementDataSize(cgid, 1, z, 1, &elementDataSize);CHKERRQ(ierr);
ierr = PetscMalloc1(elementDataSize, &elements);CHKERRQ(ierr);
ierr = cg_elements_read(cgid, 1, z, 1, elements, NULL);CHKERRQ(ierr);
for (c_loc = start, off = 0; c_loc <= end; ++c_loc, ++c) {
switch (elements[off]) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) elements[off]);
}
ierr = DMPlexSetConeSize(*dm, c, numCorners);CHKERRQ(ierr);
off += numCorners+1;
}
ierr = PetscFree(elements);CHKERRQ(ierr);
} else {
switch (cellType) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) cellType);
}
for (c_loc = start; c_loc <= end; ++c_loc, ++c) {
ierr = DMPlexSetConeSize(*dm, c, numCorners);CHKERRQ(ierr);
}
}
}
ierr = DMSetUp(*dm);CHKERRQ(ierr);
for (z = 1, c = 0; z <= nzones; ++z) {
ElementType_t cellType;
cgsize_t *elements, elementDataSize, start, end;
int nbndry, parentFlag;
PetscInt *cone, numc, numCorners, maxCorners = 27;
ierr = cg_section_read(cgid, 1, z, 1, buffer, &cellType, &start, &end, &nbndry, &parentFlag);CHKERRQ(ierr);
numc = end - start;
/* This alone is reason enough to bludgeon every single CGNDS developer, this must be what they describe as the "idiocy of crowds" */
ierr = cg_ElementDataSize(cgid, 1, z, 1, &elementDataSize);CHKERRQ(ierr);
ierr = PetscMalloc2(elementDataSize,&elements,maxCorners,&cone);CHKERRQ(ierr);
ierr = cg_elements_read(cgid, 1, z, 1, elements, NULL);CHKERRQ(ierr);
if (cellType == MIXED) {
/* CGNS uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc <= numc; ++c_loc, ++c) {
switch (elements[v]) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) elements[v]);
}
++v;
for (v_loc = 0; v_loc < numCorners; ++v_loc, ++v) {
cone[v_loc] = elements[v]+numCells-1;
}
/* Tetrahedra are inverted */
if (elements[v] == TETRA_4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted */
if (elements[v] == HEXA_8) {
PetscInt tmp = cone[5];
cone[5] = cone[7];
cone[7] = tmp;
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(*dm, "zone", c, z);CHKERRQ(ierr);
}
} else {
switch (cellType) {
case TRI_3: numCorners = 3;break;
case QUAD_4: numCorners = 4;break;
case TETRA_4: numCorners = 4;break;
case HEXA_8: numCorners = 8;break;
default: SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid cell type %d", (int) cellType);
}
/* CGNS uses Fortran-based indexing, sieve uses C-style and numbers cell first then vertices. */
for (c_loc = 0, v = 0; c_loc <= numc; ++c_loc, ++c) {
for (v_loc = 0; v_loc < numCorners; ++v_loc, ++v) {
cone[v_loc] = elements[v]+numCells-1;
}
/* Tetrahedra are inverted */
if (cellType == TETRA_4) {
PetscInt tmp = cone[0];
cone[0] = cone[1];
cone[1] = tmp;
}
/* Hexahedra are inverted, and they give the top first */
if (cellType == HEXA_8) {
PetscInt tmp = cone[5];
cone[5] = cone[7];
cone[7] = tmp;
}
ierr = DMPlexSetCone(*dm, c, cone);CHKERRQ(ierr);
ierr = DMPlexSetLabelValue(*dm, "zone", c, z);CHKERRQ(ierr);
}
}
ierr = PetscFree2(elements,cone);CHKERRQ(ierr);
}
}
ierr = DMPlexSymmetrize(*dm);CHKERRQ(ierr);
ierr = DMPlexStratify(*dm);CHKERRQ(ierr);
if (interpolate) {
DM idm = NULL;
ierr = DMPlexInterpolate(*dm, &idm);CHKERRQ(ierr);
/* Maintain zone label */
{
DMLabel label;
ierr = DMPlexRemoveLabel(*dm, "zone", &label);CHKERRQ(ierr);
if (label) {ierr = DMPlexAddLabel(idm, label);CHKERRQ(ierr);}
}
ierr = DMDestroy(dm);CHKERRQ(ierr);
*dm = idm;
}
/* Read coordinates */
ierr = DMGetCoordinateSection(*dm, &coordSection);CHKERRQ(ierr);
ierr = PetscSectionSetNumFields(coordSection, 1);CHKERRQ(ierr);
ierr = PetscSectionSetFieldComponents(coordSection, 0, dim);CHKERRQ(ierr);
ierr = PetscSectionSetChart(coordSection, numCells, numCells + numVertices);CHKERRQ(ierr);
for (v = numCells; v < numCells+numVertices; ++v) {
ierr = PetscSectionSetDof(coordSection, v, dim);CHKERRQ(ierr);
ierr = PetscSectionSetFieldDof(coordSection, v, 0, dim);CHKERRQ(ierr);
}
ierr = PetscSectionSetUp(coordSection);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(coordSection, &coordSize);CHKERRQ(ierr);
ierr = VecCreate(comm, &coordinates);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) coordinates, "coordinates");CHKERRQ(ierr);
ierr = VecSetSizes(coordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
ierr = VecSetType(coordinates,VECSTANDARD);CHKERRQ(ierr);
ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
if (!rank) {
PetscInt off = 0;
float *x[3];
int z, d;
ierr = PetscMalloc3(numVertices,&x[0],numVertices,&x[1],numVertices,&x[2]);CHKERRQ(ierr);
for (z = 1; z <= nzones; ++z) {
DataType_t datatype;
cgsize_t sizes[3]; /* Number of vertices, number of cells, number of boundary vertices */
cgsize_t range_min[3] = {1, 1, 1};
cgsize_t range_max[3] = {1, 1, 1};
int ngrids, ncoords;
ierr = cg_zone_read(cgid, 1, z, buffer, sizes);CHKERRQ(ierr);
range_max[0] = sizes[0];
ierr = cg_ngrids(cgid, 1, z, &ngrids);CHKERRQ(ierr);
if (ngrids > 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a single grid, not %d\n",ngrids);
ierr = cg_ncoords(cgid, 1, z, &ncoords);CHKERRQ(ierr);
if (ncoords != dim) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"CGNS file must have a coordinate array for each dimension, not %d\n",ncoords);
for (d = 0; d < dim; ++d) {
ierr = cg_coord_info(cgid, 1, z, 1+d, &datatype, buffer);CHKERRQ(ierr);
ierr = cg_coord_read(cgid, 1, z, buffer, RealSingle, range_min, range_max, x[d]);CHKERRQ(ierr);
}
if (dim > 0) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+0] = x[0][v];
}
if (dim > 1) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+1] = x[1][v];
}
if (dim > 2) {
for (v = 0; v < sizes[0]; ++v) coords[(v+off)*dim+2] = x[2][v];
}
off += sizes[0];
}
ierr = PetscFree3(x[0],x[1],x[2]);CHKERRQ(ierr);
}
ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
ierr = DMSetCoordinatesLocal(*dm, coordinates);CHKERRQ(ierr);
ierr = VecDestroy(&coordinates);CHKERRQ(ierr);
/* Read boundary conditions */
if (!rank) {
DMLabel label;
BCType_t bctype;
DataType_t datatype;
PointSetType_t pointtype;
cgsize_t *points;
PetscReal *normals;
int normal[3];
char *bcname = buffer;
cgsize_t npoints, nnormals;
int z, nbc, bc, c, ndatasets;
for (z = 1; z <= nzones; ++z) {
ierr = cg_nbocos(cgid, 1, z, &nbc);CHKERRQ(ierr);
for (bc = 1; bc <= nbc; ++bc) {
ierr = cg_boco_info(cgid, 1, z, bc, bcname, &bctype, &pointtype, &npoints, normal, &nnormals, &datatype, &ndatasets);CHKERRQ(ierr);
ierr = DMPlexCreateLabel(*dm, bcname);CHKERRQ(ierr);
ierr = DMPlexGetLabel(*dm, bcname, &label);CHKERRQ(ierr);
ierr = PetscMalloc2(npoints, &points, nnormals, &normals);CHKERRQ(ierr);
ierr = cg_boco_read(cgid, 1, z, bc, points, (void *) normals);CHKERRQ(ierr);
if (pointtype == ElementRange) {
/* Range of cells: assuming half-open interval since the documentation sucks */
for (c = points[0]; c < points[1]; ++c) {
ierr = DMLabelSetValue(label, c - cellStart[z-1], 1);CHKERRQ(ierr);
}
} else if (pointtype == ElementList) {
/* List of cells */
for (c = 0; c < npoints; ++c) {
ierr = DMLabelSetValue(label, points[c] - cellStart[z-1], 1);CHKERRQ(ierr);
}
} else if (pointtype == PointRange) {
GridLocation_t gridloc;
/* List of points: Oh please, someone get the CGNS developers away from a computer. This is unconscionable. */
ierr = cg_goto(cgid, 1, "Zone_t", z, "BC_t", bc, "end");CHKERRQ(ierr);
ierr = cg_gridlocation_read(&gridloc);CHKERRQ(ierr);
/* Range of points: assuming half-open interval since the documentation sucks */
for (c = points[0]; c < points[1]; ++c) {
if (gridloc == Vertex) {ierr = DMLabelSetValue(label, c - vertStart[z-1], 1);CHKERRQ(ierr);}
else {ierr = DMLabelSetValue(label, c - cellStart[z-1], 1);CHKERRQ(ierr);}
}
} else if (pointtype == PointList) {
GridLocation_t gridloc;
/* List of points: Oh please, someone get the CGNS developers away from a computer. This is unconscionable. */
ierr = cg_goto(cgid, 1, "Zone_t", z, "BC_t", bc, "end");
ierr = cg_gridlocation_read(&gridloc);
for (c = 0; c < npoints; ++c) {
if (gridloc == Vertex) {ierr = DMLabelSetValue(label, points[c] - vertStart[z-1], 1);CHKERRQ(ierr);}
else {ierr = DMLabelSetValue(label, points[c] - cellStart[z-1], 1);CHKERRQ(ierr);}
}
} else SETERRQ1(comm, PETSC_ERR_SUP, "Unsupported point set type %d", (int) pointtype);
ierr = PetscFree2(points, normals);CHKERRQ(ierr);
}
}
ierr = PetscFree2(cellStart, vertStart);CHKERRQ(ierr);
}
#else
SETERRQ(comm, PETSC_ERR_SUP, "This method requires CGNS support. Reconfigure using --with-cgns-dir");
#endif
PetscFunctionReturn(0);
}
PetscErrorCode random_network(PetscInt nvertex,PetscInt *pnbranch,Node **pnode,Branch **pbranch,PetscInt **pedgelist,PetscInt seed)
{
PetscErrorCode ierr;
PetscInt i, j, nedges = 0;
PetscInt *edgelist;
PetscInt nbat, ncurr, fr, to;
PetscReal *x, *y, value, xmax = 10.0; /* generate points in square */
PetscReal maxdist = 0.0, dist, alpha, beta, prob;
PetscRandom rnd;
Branch *branch;
Node *node;
Edge *head = NULL, *nnew= NULL, *aux= NULL;
PetscFunctionBeginUser;
ierr = PetscRandomCreate(PETSC_COMM_SELF,&rnd);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rnd);CHKERRQ(ierr);
ierr = PetscRandomSetSeed(rnd, seed);CHKERRQ(ierr);
ierr = PetscRandomSeed(rnd);CHKERRQ(ierr);
/* These parameters might be modified for experimentation */
nbat = (PetscInt)(0.1*nvertex);
ncurr = (PetscInt)(0.1*nvertex);
alpha = 0.6;
beta = 0.2;
ierr = PetscMalloc2(nvertex,&x,nvertex,&y);CHKERRQ(ierr);
ierr = PetscRandomSetInterval(rnd,0.0,xmax);CHKERRQ(ierr);
for (i=0; i<nvertex; i++) {
ierr = PetscRandomGetValueReal(rnd,&x[i]);CHKERRQ(ierr);
ierr = PetscRandomGetValueReal(rnd,&y[i]);CHKERRQ(ierr);
}
/* find maximum distance */
for (i=0; i<nvertex; i++) {
for (j=0; j<nvertex; j++) {
dist = findDistance(x[i],x[j],y[i],y[j]);
if (dist >= maxdist) maxdist = dist;
}
}
ierr = PetscRandomSetInterval(rnd,0.0,1.0);CHKERRQ(ierr);
for (i=0; i<nvertex; i++) {
for (j=0; j<nvertex; j++) {
if (j != i) {
dist = findDistance(x[i],x[j],y[i],y[j]);
prob = beta*PetscExpScalar(-dist/(maxdist*alpha));
ierr = PetscRandomGetValueReal(rnd,&value);CHKERRQ(ierr);
if (value <= prob) {
ierr = PetscMalloc1(1,&nnew);CHKERRQ(ierr);
if (head == NULL) {
head = nnew;
head->next = NULL;
head->n = nedges;
head->i = i;
head->j = j;
} else {
aux = head;
head = nnew;
head->n = nedges;
head->next = aux;
head->i = i;
head->j = j;
}
nedges += 1;
}
}
}
}
ierr = PetscMalloc1(2*nedges,&edgelist);CHKERRQ(ierr);
for (aux = head; aux; aux = aux->next) {
edgelist[(aux->n)*2] = aux->i;
edgelist[(aux->n)*2 + 1] = aux->j;
}
aux = head;
while (aux != NULL) {
nnew = aux;
aux = aux->next;
ierr = PetscFree(nnew);CHKERRQ(ierr);
}
ierr = PetscCalloc2(nvertex,&node,nedges,&branch);CHKERRQ(ierr);
for (i = 0; i < nvertex; i++) {
node[i].id = i;
node[i].inj = 0;
node[i].gr = PETSC_FALSE;
}
for (i = 0; i < nedges; i++) {
branch[i].id = i;
branch[i].r = 1.0;
branch[i].bat = 0;
}
/* Chose random node as ground voltage */
ierr = PetscRandomSetInterval(rnd,0.0,nvertex);CHKERRQ(ierr);
ierr = PetscRandomGetValueReal(rnd,&value);CHKERRQ(ierr);
node[(int)value].gr = PETSC_TRUE;
/* Create random current and battery injectionsa */
for (i=0; i<ncurr; i++) {
ierr = PetscRandomSetInterval(rnd,0.0,nvertex);CHKERRQ(ierr);
ierr = PetscRandomGetValueReal(rnd,&value);CHKERRQ(ierr);
fr = edgelist[(int)value*2];
to = edgelist[(int)value*2 + 1];
node[fr].inj += 1.0;
node[to].inj -= 1.0;
}
for (i=0; i<nbat; i++) {
ierr = PetscRandomSetInterval(rnd,0.0,nedges);CHKERRQ(ierr);
ierr = PetscRandomGetValueReal(rnd,&value);CHKERRQ(ierr);
branch[(int)value].bat += 1.0;
}
ierr = PetscFree2(x,y);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rnd);CHKERRQ(ierr);
/* assign pointers */
*pnbranch = nedges;
*pedgelist = edgelist;
*pbranch = branch;
*pnode = node;
PetscFunctionReturn(ierr);
}
/*@
DMPlexOrient - Give a consistent orientation to the input mesh
Input Parameters:
. dm - The DM
Note: The orientation data for the DM are change in-place.
$ This routine will fail for non-orientable surfaces, such as the Moebius strip.
Level: advanced
.seealso: DMCreate(), DMPLEX
@*/
PetscErrorCode DMPlexOrient(DM dm)
{
MPI_Comm comm;
PetscSF sf;
const PetscInt *lpoints;
const PetscSFNode *rpoints;
PetscSFNode *rorntComp = NULL, *lorntComp = NULL;
PetscInt *numNeighbors, **neighbors;
PetscSFNode *nrankComp;
PetscBool *match, *flipped;
PetscBT seenCells, flippedCells, seenFaces;
PetscInt *faceFIFO, fTop, fBottom, *cellComp, *faceComp;
PetscInt numLeaves, numRoots, dim, h, cStart, cEnd, c, cell, fStart, fEnd, face, off, totNeighbors = 0;
PetscMPIInt rank, size, numComponents, comp = 0;
PetscBool flg, flg2;
PetscViewer viewer = NULL, selfviewer = NULL;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view", &flg);CHKERRQ(ierr);
ierr = PetscOptionsHasName(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-orientation_view_synchronized", &flg2);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &numRoots, &numLeaves, &lpoints, &rpoints);CHKERRQ(ierr);
/* Truth Table
mismatch flips do action mismatch flipA ^ flipB action
F 0 flips no F F F
F 1 flip yes F T T
F 2 flips no T F T
T 0 flips yes T T F
T 1 flip no
T 2 flips yes
*/
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMPlexGetVTKCellHeight(dm, &h);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h, &cStart, &cEnd);CHKERRQ(ierr);
ierr = DMPlexGetHeightStratum(dm, h+1, &fStart, &fEnd);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &seenCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
ierr = PetscBTCreate(cEnd - cStart, &flippedCells);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, flippedCells);CHKERRQ(ierr);
ierr = PetscBTCreate(fEnd - fStart, &seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscCalloc3(fEnd - fStart, &faceFIFO, cEnd-cStart, &cellComp, fEnd-fStart, &faceComp);CHKERRQ(ierr);
/*
OLD STYLE
- Add an integer array over cells and faces (component) for connected component number
Foreach component
- Mark the initial cell as seen
- Process component as usual
- Set component for all seenCells
- Wipe seenCells and seenFaces (flippedCells can stay)
- Generate parallel adjacency for component using SF and seenFaces
- Collect numComponents adj data from each proc to 0
- Build same serial graph
- Use same solver
- Use Scatterv to to send back flipped flags for each component
- Negate flippedCells by component
NEW STYLE
- Create the adj on each process
- Bootstrap to complete graph on proc 0
*/
/* Loop over components */
for (cell = cStart; cell < cEnd; ++cell) cellComp[cell-cStart] = -1;
do {
/* Look for first unmarked cell */
for (cell = cStart; cell < cEnd; ++cell) if (cellComp[cell-cStart] < 0) break;
if (cell >= cEnd) break;
/* Initialize FIFO with first cell in component */
{
const PetscInt *cone;
PetscInt coneSize;
fTop = fBottom = 0;
ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) {
faceFIFO[fBottom++] = cone[c];
ierr = PetscBTSet(seenFaces, cone[c]-fStart);CHKERRQ(ierr);
}
ierr = PetscBTSet(seenCells, cell-cStart);CHKERRQ(ierr);
}
/* Consider each face in FIFO */
while (fTop < fBottom) {
ierr = DMPlexCheckFace_Internal(dm, faceFIFO, &fTop, &fBottom, cStart, fStart, fEnd, seenCells, flippedCells, seenFaces);CHKERRQ(ierr);
}
/* Set component for cells and faces */
for (cell = 0; cell < cEnd-cStart; ++cell) {
if (PetscBTLookup(seenCells, cell)) cellComp[cell] = comp;
}
for (face = 0; face < fEnd-fStart; ++face) {
if (PetscBTLookup(seenFaces, face)) faceComp[face] = comp;
}
/* Wipe seenCells and seenFaces for next component */
ierr = PetscBTMemzero(fEnd - fStart, seenFaces);CHKERRQ(ierr);
ierr = PetscBTMemzero(cEnd - cStart, seenCells);CHKERRQ(ierr);
++comp;
} while (1);
numComponents = comp;
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for serial flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Now all subdomains are oriented, but we need a consistent parallel orientation */
if (numLeaves >= 0) {
/* Store orientations of boundary faces*/
ierr = PetscCalloc2(numRoots,&rorntComp,numRoots,&lorntComp);CHKERRQ(ierr);
for (face = fStart; face < fEnd; ++face) {
const PetscInt *cone, *support, *ornt;
PetscInt coneSize, supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) continue;
ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
ierr = DMPlexGetCone(dm, support[0], &cone);CHKERRQ(ierr);
ierr = DMPlexGetConeSize(dm, support[0], &coneSize);CHKERRQ(ierr);
ierr = DMPlexGetConeOrientation(dm, support[0], &ornt);CHKERRQ(ierr);
for (c = 0; c < coneSize; ++c) if (cone[c] == face) break;
if (dim == 1) {
/* Use cone position instead, shifted to -1 or 1 */
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = 1-c*2;
else rorntComp[face].rank = c*2-1;
} else {
if (PetscBTLookup(flippedCells, support[0]-cStart)) rorntComp[face].rank = ornt[c] < 0 ? -1 : 1;
else rorntComp[face].rank = ornt[c] < 0 ? 1 : -1;
}
rorntComp[face].index = faceComp[face-fStart];
}
/* Communicate boundary edge orientations */
ierr = PetscSFBcastBegin(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf, MPIU_2INT, rorntComp, lorntComp);CHKERRQ(ierr);
}
/* Get process adjacency */
ierr = PetscMalloc2(numComponents, &numNeighbors, numComponents, &neighbors);CHKERRQ(ierr);
viewer = PETSC_VIEWER_STDOUT_(PetscObjectComm((PetscObject)dm));
if (flg2) {ierr = PetscViewerASCIIPushSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
for (comp = 0; comp < numComponents; ++comp) {
PetscInt l, n;
numNeighbors[comp] = 0;
ierr = PetscMalloc1(PetscMax(numLeaves, 0), &neighbors[comp]);CHKERRQ(ierr);
/* I know this is p^2 time in general, but for bounded degree its alright */
for (l = 0; l < numLeaves; ++l) {
const PetscInt face = lpoints[l];
/* Find a representative face (edge) separating pairs of procs */
if ((face >= fStart) && (face < fEnd) && (faceComp[face-fStart] == comp)) {
const PetscInt rrank = rpoints[l].rank;
const PetscInt rcomp = lorntComp[face].index;
for (n = 0; n < numNeighbors[comp]; ++n) if ((rrank == rpoints[neighbors[comp][n]].rank) && (rcomp == lorntComp[lpoints[neighbors[comp][n]]].index)) break;
if (n >= numNeighbors[comp]) {
PetscInt supportSize;
ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
if (supportSize != 1) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Boundary faces should see one cell, not %d", supportSize);
if (flg) {ierr = PetscViewerASCIIPrintf(selfviewer, "[%d]: component %d, Found representative leaf %d (face %d) connecting to face %d on (%d, %d) with orientation %d\n", rank, comp, l, face, rpoints[l].index, rrank, rcomp, lorntComp[face].rank);CHKERRQ(ierr);}
neighbors[comp][numNeighbors[comp]++] = l;
}
}
}
totNeighbors += numNeighbors[comp];
}
ierr = PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
if (flg2) {ierr = PetscViewerASCIIPopSynchronized(viewer);CHKERRQ(ierr);}
ierr = PetscMalloc2(totNeighbors, &nrankComp, totNeighbors, &match);CHKERRQ(ierr);
for (comp = 0, off = 0; comp < numComponents; ++comp) {
PetscInt n;
for (n = 0; n < numNeighbors[comp]; ++n, ++off) {
const PetscInt face = lpoints[neighbors[comp][n]];
const PetscInt o = rorntComp[face].rank*lorntComp[face].rank;
if (o < 0) match[off] = PETSC_TRUE;
else if (o > 0) match[off] = PETSC_FALSE;
else SETERRQ5(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid face %d (%d, %d) neighbor: %d comp: %d", face, rorntComp[face], lorntComp[face], neighbors[comp][n], comp);
nrankComp[off].rank = rpoints[neighbors[comp][n]].rank;
nrankComp[off].index = lorntComp[lpoints[neighbors[comp][n]]].index;
}
ierr = PetscFree(neighbors[comp]);CHKERRQ(ierr);
}
/* Collect the graph on 0 */
if (numLeaves >= 0) {
Mat G;
PetscBT seenProcs, flippedProcs;
PetscInt *procFIFO, pTop, pBottom;
PetscInt *N = NULL, *Noff;
PetscSFNode *adj = NULL;
PetscBool *val = NULL;
PetscMPIInt *recvcounts = NULL, *displs = NULL, *Nc, p, o;
PetscMPIInt size = 0;
ierr = PetscCalloc1(numComponents, &flipped);CHKERRQ(ierr);
if (!rank) {ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);}
ierr = PetscCalloc4(size, &recvcounts, size+1, &displs, size, &Nc, size+1, &Noff);CHKERRQ(ierr);
ierr = MPI_Gather(&numComponents, 1, MPI_INT, Nc, 1, MPI_INT, 0, comm);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
if (!rank) {ierr = PetscMalloc1(displs[size],&N);CHKERRQ(ierr);}
ierr = MPI_Gatherv(numNeighbors, numComponents, MPIU_INT, N, Nc, displs, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 0, o = 0; p < size; ++p) {
recvcounts[p] = 0;
for (c = 0; c < Nc[p]; ++c, ++o) recvcounts[p] += N[o];
displs[p+1] = displs[p] + recvcounts[p];
}
if (!rank) {ierr = PetscMalloc2(displs[size], &adj, displs[size], &val);CHKERRQ(ierr);}
ierr = MPI_Gatherv(nrankComp, totNeighbors, MPIU_2INT, adj, recvcounts, displs, MPIU_2INT, 0, comm);CHKERRQ(ierr);
ierr = MPI_Gatherv(match, totNeighbors, MPIU_BOOL, val, recvcounts, displs, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
if (!rank) {
for (p = 1; p <= size; ++p) {Noff[p] = Noff[p-1] + Nc[p-1];}
if (flg) {
PetscInt n;
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
ierr = PetscPrintf(PETSC_COMM_SELF, "Proc %d Comp %d:\n", p, c);CHKERRQ(ierr);
for (n = 0; n < N[Noff[p]+c]; ++n, ++off) {
ierr = PetscPrintf(PETSC_COMM_SELF, " edge (%d, %d) (%d):\n", adj[off].rank, adj[off].index, val[off]);CHKERRQ(ierr);
}
}
}
}
/* Symmetrize the graph */
ierr = MatCreate(PETSC_COMM_SELF, &G);CHKERRQ(ierr);
ierr = MatSetSizes(G, Noff[size], Noff[size], Noff[size], Noff[size]);CHKERRQ(ierr);
ierr = MatSetUp(G);CHKERRQ(ierr);
for (p = 0, off = 0; p < size; ++p) {
for (c = 0; c < Nc[p]; ++c) {
const PetscInt r = Noff[p]+c;
PetscInt n;
for (n = 0; n < N[r]; ++n, ++off) {
const PetscInt q = Noff[adj[off].rank] + adj[off].index;
const PetscScalar o = val[off] ? 1.0 : 0.0;
ierr = MatSetValues(G, 1, &r, 1, &q, &o, INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(G, 1, &q, 1, &r, &o, INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &seenProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], seenProcs);CHKERRQ(ierr);
ierr = PetscBTCreate(Noff[size], &flippedProcs);CHKERRQ(ierr);
ierr = PetscBTMemzero(Noff[size], flippedProcs);CHKERRQ(ierr);
ierr = PetscMalloc1(Noff[size], &procFIFO);CHKERRQ(ierr);
pTop = pBottom = 0;
for (p = 0; p < Noff[size]; ++p) {
if (PetscBTLookup(seenProcs, p)) continue;
/* Initialize FIFO with next proc */
procFIFO[pBottom++] = p;
ierr = PetscBTSet(seenProcs, p);CHKERRQ(ierr);
/* Consider each proc in FIFO */
while (pTop < pBottom) {
const PetscScalar *ornt;
const PetscInt *neighbors;
PetscInt proc, nproc, seen, flippedA, flippedB, mismatch, numNeighbors, n;
proc = procFIFO[pTop++];
flippedA = PetscBTLookup(flippedProcs, proc) ? 1 : 0;
ierr = MatGetRow(G, proc, &numNeighbors, &neighbors, &ornt);CHKERRQ(ierr);
/* Loop over neighboring procs */
for (n = 0; n < numNeighbors; ++n) {
nproc = neighbors[n];
mismatch = PetscRealPart(ornt[n]) > 0.5 ? 0 : 1;
seen = PetscBTLookup(seenProcs, nproc);
flippedB = PetscBTLookup(flippedProcs, nproc) ? 1 : 0;
if (mismatch ^ (flippedA ^ flippedB)) {
if (seen) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Previously seen procs %d and %d do not match: Fault mesh is non-orientable", proc, nproc);
if (!flippedB) {
ierr = PetscBTSet(flippedProcs, nproc);CHKERRQ(ierr);
} else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Inconsistent mesh orientation: Fault mesh is non-orientable");
} else if (mismatch && flippedA && flippedB) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Attempt to flip already flipped cell: Fault mesh is non-orientable");
if (!seen) {
procFIFO[pBottom++] = nproc;
ierr = PetscBTSet(seenProcs, nproc);CHKERRQ(ierr);
}
}
}
}
ierr = PetscFree(procFIFO);CHKERRQ(ierr);
ierr = MatDestroy(&G);CHKERRQ(ierr);
ierr = PetscFree2(adj, val);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenProcs);CHKERRQ(ierr);
}
/* Scatter flip flags */
{
PetscBool *flips = NULL;
if (!rank) {
ierr = PetscMalloc1(Noff[size], &flips);CHKERRQ(ierr);
for (p = 0; p < Noff[size]; ++p) {
flips[p] = PetscBTLookup(flippedProcs, p) ? PETSC_TRUE : PETSC_FALSE;
if (flg && flips[p]) {ierr = PetscPrintf(comm, "Flipping Proc+Comp %d:\n", p);CHKERRQ(ierr);}
}
for (p = 0; p < size; ++p) {
displs[p+1] = displs[p] + Nc[p];
}
}
ierr = MPI_Scatterv(flips, Nc, displs, MPIU_BOOL, flipped, numComponents, MPIU_BOOL, 0, comm);CHKERRQ(ierr);
ierr = PetscFree(flips);CHKERRQ(ierr);
}
if (!rank) {ierr = PetscBTDestroy(&flippedProcs);CHKERRQ(ierr);}
ierr = PetscFree(N);CHKERRQ(ierr);
ierr = PetscFree4(recvcounts, displs, Nc, Noff);CHKERRQ(ierr);
ierr = PetscFree2(nrankComp, match);CHKERRQ(ierr);
/* Decide whether to flip cells in each component */
for (c = 0; c < cEnd-cStart; ++c) {if (flipped[cellComp[c]]) {ierr = PetscBTNegate(flippedCells, c);CHKERRQ(ierr);}}
ierr = PetscFree(flipped);CHKERRQ(ierr);
}
if (flg) {
PetscViewer v;
ierr = PetscViewerASCIIGetStdout(comm, &v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(v);CHKERRQ(ierr);
ierr = PetscViewerASCIISynchronizedPrintf(v, "[%d]BT for parallel flipped cells:\n", rank);CHKERRQ(ierr);
ierr = PetscBTView(cEnd-cStart, flippedCells, v);CHKERRQ(ierr);
ierr = PetscViewerFlush(v);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(v);CHKERRQ(ierr);
}
/* Reverse flipped cells in the mesh */
for (c = cStart; c < cEnd; ++c) {
if (PetscBTLookup(flippedCells, c-cStart)) {
ierr = DMPlexReverseCell(dm, c);CHKERRQ(ierr);
}
}
ierr = PetscBTDestroy(&seenCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&flippedCells);CHKERRQ(ierr);
ierr = PetscBTDestroy(&seenFaces);CHKERRQ(ierr);
ierr = PetscFree2(numNeighbors, neighbors);CHKERRQ(ierr);
ierr = PetscFree2(rorntComp, lorntComp);CHKERRQ(ierr);
ierr = PetscFree3(faceFIFO, cellComp, faceComp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode ComputeSpectral(DM dm, Vec u, PetscInt numPlanes, const PetscInt planeDir[], const PetscReal planeCoord[], AppCtx *user)
{
MPI_Comm comm;
PetscSection coordSection, section;
Vec coordinates, uloc;
const PetscScalar *coords, *array;
PetscInt p;
PetscMPIInt size, rank;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = PetscObjectGetComm((PetscObject) dm, &comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = DMGetLocalVector(dm, &uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalBegin(dm, u, INSERT_VALUES, uloc);CHKERRQ(ierr);
ierr = DMGlobalToLocalEnd(dm, u, INSERT_VALUES, uloc);CHKERRQ(ierr);
ierr = DMPlexInsertBoundaryValues(dm, PETSC_TRUE, uloc, 0.0, NULL, NULL, NULL);CHKERRQ(ierr);
ierr = VecViewFromOptions(uloc, NULL, "-sol_view");CHKERRQ(ierr);
ierr = DMGetDefaultSection(dm, §ion);CHKERRQ(ierr);
ierr = VecGetArrayRead(uloc, &array);CHKERRQ(ierr);
ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
ierr = VecGetArrayRead(coordinates, &coords);CHKERRQ(ierr);
for (p = 0; p < numPlanes; ++p) {
DMLabel label;
char name[PETSC_MAX_PATH_LEN];
Mat F;
Vec x, y;
IS stratum;
PetscReal *ray, *gray;
PetscScalar *rvals, *svals, *gsvals;
PetscInt *perm, *nperm;
PetscInt n, N, i, j, off, offu;
const PetscInt *points;
ierr = PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "spectral_plane_%D", p);CHKERRQ(ierr);
ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
ierr = DMLabelGetStratumIS(label, 1, &stratum);CHKERRQ(ierr);
ierr = ISGetLocalSize(stratum, &n);CHKERRQ(ierr);
ierr = ISGetIndices(stratum, &points);CHKERRQ(ierr);
ierr = PetscMalloc2(n, &ray, n, &svals);CHKERRQ(ierr);
for (i = 0; i < n; ++i) {
ierr = PetscSectionGetOffset(coordSection, points[i], &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, points[i], &offu);CHKERRQ(ierr);
ray[i] = PetscRealPart(coords[off+((planeDir[p]+1)%2)]);
svals[i] = array[offu];
}
/* Gather the ray data to proc 0 */
if (size > 1) {
PetscMPIInt *cnt, *displs, p;
ierr = PetscCalloc2(size, &cnt, size, &displs);CHKERRQ(ierr);
ierr = MPI_Gather(&n, 1, MPIU_INT, cnt, 1, MPIU_INT, 0, comm);CHKERRQ(ierr);
for (p = 1; p < size; ++p) displs[p] = displs[p-1] + cnt[p-1];
N = displs[size-1] + cnt[size-1];
ierr = PetscMalloc2(N, &gray, N, &gsvals);CHKERRQ(ierr);
ierr = MPI_Gatherv(ray, n, MPIU_REAL, gray, cnt, displs, MPIU_REAL, 0, comm);CHKERRQ(ierr);
ierr = MPI_Gatherv(svals, n, MPIU_SCALAR, gsvals, cnt, displs, MPIU_SCALAR, 0, comm);CHKERRQ(ierr);
ierr = PetscFree2(cnt, displs);CHKERRQ(ierr);
} else {
N = n;
gray = ray;
gsvals = svals;
}
if (!rank) {
/* Sort point along ray */
ierr = PetscMalloc2(N, &perm, N, &nperm);CHKERRQ(ierr);
for (i = 0; i < N; ++i) {perm[i] = i;}
ierr = PetscSortRealWithPermutation(N, gray, perm);CHKERRQ(ierr);
/* Count duplicates and squish mapping */
nperm[0] = perm[0];
for (i = 1, j = 1; i < N; ++i) {
if (PetscAbsReal(gray[perm[i]] - gray[perm[i-1]]) > PETSC_SMALL) nperm[j++] = perm[i];
}
/* Create FFT structs */
ierr = MatCreateFFT(PETSC_COMM_SELF, 1, &j, MATFFTW, &F);CHKERRQ(ierr);
ierr = MatCreateVecs(F, &x, &y);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) y, name);CHKERRQ(ierr);
ierr = VecGetArray(x, &rvals);CHKERRQ(ierr);
for (i = 0, j = 0; i < N; ++i) {
if (i > 0 && PetscAbsReal(gray[perm[i]] - gray[perm[i-1]]) < PETSC_SMALL) continue;
rvals[j] = gsvals[nperm[j]];
++j;
}
ierr = PetscFree2(perm, nperm);CHKERRQ(ierr);
if (size > 1) {ierr = PetscFree2(gray, gsvals);CHKERRQ(ierr);}
ierr = VecRestoreArray(x, &rvals);CHKERRQ(ierr);
/* Do FFT along the ray */
ierr = MatMult(F, x, y);CHKERRQ(ierr);
/* Chop FFT */
ierr = VecChop(y, PETSC_SMALL);CHKERRQ(ierr);
ierr = VecViewFromOptions(x, NULL, "-real_view");CHKERRQ(ierr);
ierr = VecViewFromOptions(y, NULL, "-fft_view");CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = MatDestroy(&F);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(stratum, &points);CHKERRQ(ierr);
ierr = ISDestroy(&stratum);CHKERRQ(ierr);
ierr = PetscFree2(ray, svals);CHKERRQ(ierr);
}
ierr = VecRestoreArrayRead(coordinates, &coords);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(uloc, &array);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &uloc);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
AppCtx ctx;
PetscErrorCode (**funcs)(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nf, PetscScalar *u, void *ctx);
DM dm;
PetscFE fe;
DMInterpolationInfo interpolator;
Vec lu, fieldVals;
PetscScalar *vals;
const PetscScalar *ivals, *vcoords;
PetscReal *pcoords;
PetscBool pointsAllProcs=PETSC_TRUE;
PetscInt spaceDim, c, Np, p;
PetscMPIInt rank, size;
PetscViewer selfviewer;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc, &argv, NULL,help);if (ierr) return ierr;
ierr = ProcessOptions(PETSC_COMM_WORLD, &ctx);CHKERRQ(ierr);
ierr = CreateMesh(PETSC_COMM_WORLD, &ctx, &dm);CHKERRQ(ierr);
ierr = DMGetCoordinateDim(dm, &spaceDim);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD, &rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD, &size);CHKERRQ(ierr);
/* Create points */
ierr = CreatePoints(dm, &Np, &pcoords, &pointsAllProcs, &ctx);CHKERRQ(ierr);
/* Create interpolator */
ierr = DMInterpolationCreate(PETSC_COMM_WORLD, &interpolator);CHKERRQ(ierr);
ierr = DMInterpolationSetDim(interpolator, spaceDim);CHKERRQ(ierr);
ierr = DMInterpolationAddPoints(interpolator, Np, pcoords);CHKERRQ(ierr);
ierr = DMInterpolationSetUp(interpolator, dm, pointsAllProcs);CHKERRQ(ierr);
/* Check locations */
for (c = 0; c < interpolator->n; ++c) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d]Point %D is in Cell %D\n", rank, c, interpolator->cells[c]);CHKERRQ(ierr);
}
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD, NULL);CHKERRQ(ierr);
ierr = VecView(interpolator->coords, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Setup Discretization */
ierr = PetscFECreateDefault(PetscObjectComm((PetscObject) dm), ctx.dim, Nc, ctx.cellSimplex, NULL, -1, &fe);CHKERRQ(ierr);
ierr = DMSetField(dm, 0, NULL, (PetscObject) fe);CHKERRQ(ierr);
ierr = DMCreateDS(dm);CHKERRQ(ierr);
ierr = PetscFEDestroy(&fe);CHKERRQ(ierr);
/* Create function */
ierr = PetscCalloc2(Nc, &funcs, Nc, &vals);CHKERRQ(ierr);
for (c = 0; c < Nc; ++c) funcs[c] = linear;
ierr = DMGetLocalVector(dm, &lu);CHKERRQ(ierr);
ierr = DMProjectFunctionLocal(dm, 0.0, funcs, NULL, INSERT_ALL_VALUES, lu);CHKERRQ(ierr);
ierr = PetscViewerASCIIPushSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerGetSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(selfviewer, "[%d]solution\n", rank);CHKERRQ(ierr);
ierr = VecView(lu,selfviewer);CHKERRQ(ierr);
ierr = PetscViewerRestoreSubViewer(PETSC_VIEWER_STDOUT_WORLD,PETSC_COMM_SELF,&selfviewer);CHKERRQ(ierr);
ierr = PetscViewerFlush(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerASCIIPopSynchronized(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* Check interpolant */
ierr = VecCreateSeq(PETSC_COMM_SELF, interpolator->n * Nc, &fieldVals);CHKERRQ(ierr);
ierr = DMInterpolationSetDof(interpolator, Nc);CHKERRQ(ierr);
ierr = DMInterpolationEvaluate(interpolator, dm, lu, fieldVals);CHKERRQ(ierr);
for (p = 0; p < size; ++p) {
if (p == rank) {
ierr = PetscPrintf(PETSC_COMM_SELF, "[%d]Field values\n", rank);CHKERRQ(ierr);
ierr = VecView(fieldVals, PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
ierr = PetscBarrier((PetscObject) dm);CHKERRQ(ierr);
}
ierr = VecGetArrayRead(interpolator->coords, &vcoords);CHKERRQ(ierr);
ierr = VecGetArrayRead(fieldVals, &ivals);CHKERRQ(ierr);
for (p = 0; p < interpolator->n; ++p) {
for (c = 0; c < Nc; ++c) {
#if defined(PETSC_USE_COMPLEX)
PetscReal vcoordsReal[3];
PetscInt i;
for (i = 0; i < spaceDim; i++) vcoordsReal[i] = PetscRealPart(vcoords[p * spaceDim + i]);
#else
const PetscReal *vcoordsReal = &vcoords[p*spaceDim];
#endif
(*funcs[c])(ctx.dim, 0.0, vcoordsReal, 1, vals, NULL);
if (PetscAbsScalar(ivals[p*Nc+c] - vals[c]) > PETSC_SQRT_MACHINE_EPSILON)
SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid interpolated value %g != %g (%D, %D)", (double) PetscRealPart(ivals[p*Nc+c]), (double) PetscRealPart(vals[c]), p, c);
}
}
ierr = VecRestoreArrayRead(interpolator->coords, &vcoords);CHKERRQ(ierr);
ierr = VecRestoreArrayRead(fieldVals, &ivals);CHKERRQ(ierr);
/* Cleanup */
ierr = PetscFree(pcoords);CHKERRQ(ierr);
ierr = PetscFree2(funcs, vals);CHKERRQ(ierr);
ierr = VecDestroy(&fieldVals);CHKERRQ(ierr);
ierr = DMRestoreLocalVector(dm, &lu);CHKERRQ(ierr);
ierr = DMInterpolationDestroy(&interpolator);CHKERRQ(ierr);
ierr = DMDestroy(&dm);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
PetscErrorCode WashNetworkCreate(MPI_Comm comm,PetscInt pipesCase,Wash *wash_ptr,int **elist)
{
PetscErrorCode ierr;
PetscInt nnodes,npipes;
PetscMPIInt rank;
Wash wash;
PetscInt i,numVertices,numEdges;
int *edgelist;
Junction junctions=NULL;
Pipe pipes=NULL;
PetscFunctionBegin;
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = PetscCalloc1(1,&wash);CHKERRQ(ierr);
wash->comm = comm;
*wash_ptr = wash;
wash->Q0 = 0.477432; /* copied from initial soluiton */
wash->H0 = 150.0;
wash->HL = 143.488; /* copied from initial soluiton */
wash->nnodes_loc = 0;
numVertices = 0;
numEdges = 0;
edgelist = NULL;
if (!rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Setup pipesCase %D\n",pipesCase);CHKERRQ(ierr);
}
nnodes = 6;
ierr = PetscOptionsGetInt(NULL,NULL, "-npipenodes", &nnodes, NULL);CHKERRQ(ierr);
/* Set global number of pipes, edges, and junctions */
/*-------------------------------------------------*/
switch (pipesCase) {
case 0:
/* pipeCase 0: */
/* =============================
v0 --E0--> v1--E1--> v2 --E2-->v3
================================ */
npipes = 3;
ierr = PetscOptionsGetInt(NULL,NULL, "-npipes", &npipes, NULL);CHKERRQ(ierr);
wash->nedge = npipes;
wash->nvertex = npipes + 1;
/* Set local edges and vertices -- proc[0] sets entire network, then distributes */
numVertices = 0;
numEdges = 0;
edgelist = NULL;
if (!rank) {
numVertices = wash->nvertex;
numEdges = wash->nedge;
ierr = PetscCalloc1(2*numEdges,&edgelist);CHKERRQ(ierr);
for (i=0; i<numEdges; i++) {
edgelist[2*i] = i; edgelist[2*i+1] = i+1;
}
/* Add network components */
/*------------------------*/
ierr = PetscCalloc2(numVertices,&junctions,numEdges,&pipes);CHKERRQ(ierr);
/* vertex */
for (i=0; i<numVertices; i++) {
junctions[i].id = i;
junctions[i].isEnd = 0;
junctions[i].nedges_in = 1; junctions[i].nedges_out = 1;
/* Set GPS data */
junctions[i].latitude = 0.0;
junctions[i].longitude = 0.0;
}
junctions[0].isEnd = -1;
junctions[0].nedges_in = 0;
junctions[numVertices-1].isEnd = 1;
junctions[numVertices-1].nedges_out = 0;
/* edge and pipe */
for (i=0; i<numEdges; i++) {
pipes[i].id = i;
pipes[i].nnodes = nnodes;
}
}
break;
case 1:
/* pipeCase 1: */
/* ==========================
v2
^
|
E2
|
v0 --E0--> v3--E1--> v1
============================= */
npipes = 3;
wash->nedge = npipes;
wash->nvertex = npipes + 1;
/* Set local edges and vertices -- proc[0] sets entire network, then distributes */
if (!rank) {
numVertices = wash->nvertex;
numEdges = wash->nedge;
ierr = PetscCalloc1(2*numEdges,&edgelist);CHKERRQ(ierr);
edgelist[0] = 0; edgelist[1] = 3; /* edge[0] */
edgelist[2] = 3; edgelist[3] = 1; /* edge[1] */
edgelist[4] = 3; edgelist[5] = 2; /* edge[2] */
/* Add network components */
/*------------------------*/
ierr = PetscCalloc2(numVertices,&junctions,numEdges,&pipes);CHKERRQ(ierr);
/* vertex */
for (i=0; i<numVertices; i++) {
junctions[i].id = i;
/* Set GPS data */
junctions[i].latitude = 0.0;
junctions[i].longitude = 0.0;
}
junctions[0].isEnd = -1; junctions[0].nedges_in = 0; junctions[0].nedges_out = 1;
junctions[1].isEnd = 1; junctions[1].nedges_in = 1; junctions[1].nedges_out = 0;
junctions[2].isEnd = 1; junctions[2].nedges_in = 1; junctions[2].nedges_out = 0;
junctions[3].isEnd = 0; junctions[3].nedges_in = 1; junctions[3].nedges_out = 2;
/* edge and pipe */
for (i=0; i<numEdges; i++) {
pipes[i].id = i;
pipes[i].nnodes = nnodes;
}
}
break;
case 2:
/* pipeCase 2: */
/* ==========================
v2--> E2
|
v0 --E0--> v3--E1--> v1
============================= */
/* Set application parameters -- to be used in function evalutions */
npipes = 3;
wash->nedge = npipes;
wash->nvertex = npipes + 1;
/* Set local edges and vertices -- proc[0] sets entire network, then distributes */
if (!rank) {
numVertices = wash->nvertex;
numEdges = wash->nedge;
ierr = PetscCalloc1(2*numEdges,&edgelist);CHKERRQ(ierr);
edgelist[0] = 0; edgelist[1] = 3; /* edge[0] */
edgelist[2] = 3; edgelist[3] = 1; /* edge[1] */
edgelist[4] = 2; edgelist[5] = 3; /* edge[2] */
/* Add network components */
/*------------------------*/
ierr = PetscCalloc2(numVertices,&junctions,numEdges,&pipes);CHKERRQ(ierr);
/* vertex */
for (i=0; i<numVertices; i++) {
junctions[i].id = i;
/* Set GPS data */
junctions[i].latitude = 0.0;
junctions[i].longitude = 0.0;
}
junctions[0].isEnd = -1; junctions[0].nedges_in = 0; junctions[0].nedges_out = 1;
junctions[1].isEnd = 1; junctions[1].nedges_in = 1; junctions[1].nedges_out = 0;
junctions[2].isEnd = -1; junctions[2].nedges_in = 0; junctions[2].nedges_out = 1;
junctions[3].isEnd = 0; junctions[3].nedges_in = 2; junctions[3].nedges_out = 1;
/* edge and pipe */
for (i=0; i<numEdges; i++) {
pipes[i].id = i;
pipes[i].nnodes = nnodes;
}
}
break;
default:
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"not done yet");
}
*wash_ptr = wash;
wash->nedge = numEdges;
wash->nvertex = numVertices;
*elist = edgelist;
wash->junction = junctions;
wash->pipe = pipes;
PetscFunctionReturn(0);
}
