PetscErrorCode MatPartitioningHierarchical_ReassembleFineparts(Mat adj, IS fineparts, ISLocalToGlobalMapping mapping, IS *sfineparts)
{
PetscInt *local_indices, *global_indices,*owners,*sfineparts_indices,localsize,i;
const PetscInt *ranges,*fineparts_indices;
PetscMPIInt rank;
MPI_Comm comm;
PetscLayout rmap;
PetscSFNode *remote;
PetscSF sf;
PetscErrorCode ierr;
PetscFunctionBegin;
/*get communicator */
ierr = PetscObjectGetComm((PetscObject)adj,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MatGetLayouts(adj,&rmap,PETSC_NULL);CHKERRQ(ierr);
ierr = ISGetLocalSize(fineparts,&localsize);CHKERRQ(ierr);
ierr = PetscCalloc2(localsize,&global_indices,localsize,&local_indices);CHKERRQ(ierr);
for(i=0; i<localsize; i++){
local_indices[i] = i;
}
/*global indices */
ierr = ISLocalToGlobalMappingApply(mapping,localsize,local_indices,global_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(localsize,&owners);CHKERRQ(ierr);
/*find owners for global indices */
for(i=0; i<localsize; i++){
ierr = PetscLayoutFindOwner(rmap,global_indices[i],&owners[i]);CHKERRQ(ierr);
}
/*ranges */
ierr = PetscLayoutGetRanges(rmap,&ranges);CHKERRQ(ierr);
ierr = PetscCalloc1(ranges[rank+1]-ranges[rank],&sfineparts_indices);CHKERRQ(ierr);
ierr = ISGetIndices(fineparts,&fineparts_indices);CHKERRQ(ierr);
/*create a SF to exchange data */
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscCalloc1(localsize,&remote);CHKERRQ(ierr);
for(i=0; i<localsize; i++){
remote[i].rank = owners[i];
remote[i].index = global_indices[i]-ranges[owners[i]];
}
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
/*not sure how to add prefix to sf*/
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,localsize,localsize,PETSC_NULL,PETSC_OWN_POINTER,remote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sf,MPIU_INT,fineparts_indices,sfineparts_indices,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sf,MPIU_INT,fineparts_indices,sfineparts_indices,MPIU_REPLACE);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = ISRestoreIndices(fineparts,&fineparts_indices);CHKERRQ(ierr);
/* comm self */
ierr = ISCreateGeneral(comm,ranges[rank+1]-ranges[rank],sfineparts_indices,PETSC_OWN_POINTER,sfineparts);CHKERRQ(ierr);
ierr = PetscFree2(global_indices,local_indices);CHKERRQ(ierr);
ierr = PetscFree(owners);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode maxIndSetAgg(IS perm,Mat Gmat,PetscBool strict_aggs,PetscCoarsenData **a_locals_llist)
{
PetscErrorCode ierr;
Mat_SeqAIJ *matA,*matB=NULL;
Mat_MPIAIJ *mpimat=NULL;
MPI_Comm comm;
PetscInt num_fine_ghosts,kk,n,ix,j,*idx,*ii,iter,Iend,my0,nremoved,gid,lid,cpid,lidj,sgid,t1,t2,slid,nDone,nselected=0,state,statej;
PetscInt *cpcol_gid,*cpcol_state,*lid_cprowID,*lid_gid,*cpcol_sel_gid,*icpcol_gid,*lid_state,*lid_parent_gid=NULL;
PetscBool *lid_removed;
PetscBool isMPI,isAIJ,isOK;
const PetscInt *perm_ix;
const PetscInt nloc = Gmat->rmap->n;
PetscCoarsenData *agg_lists;
PetscLayout layout;
PetscSF sf;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Gmat,&comm);
CHKERRQ(ierr);
/* get submatrices */
ierr = PetscObjectTypeCompare((PetscObject)Gmat,MATMPIAIJ,&isMPI);
CHKERRQ(ierr);
if (isMPI) {
mpimat = (Mat_MPIAIJ*)Gmat->data;
matA = (Mat_SeqAIJ*)mpimat->A->data;
matB = (Mat_SeqAIJ*)mpimat->B->data;
/* force compressed storage of B */
ierr = MatCheckCompressedRow(mpimat->B,matB->nonzerorowcnt,&matB->compressedrow,matB->i,Gmat->rmap->n,-1.0);
CHKERRQ(ierr);
} else {
ierr = PetscObjectTypeCompare((PetscObject)Gmat,MATSEQAIJ,&isAIJ);
CHKERRQ(ierr);
matA = (Mat_SeqAIJ*)Gmat->data;
}
ierr = MatGetOwnershipRange(Gmat,&my0,&Iend);
CHKERRQ(ierr);
ierr = PetscMalloc1(nloc,&lid_gid);
CHKERRQ(ierr); /* explicit array needed */
if (mpimat) {
for (kk=0,gid=my0; kk<nloc; kk++,gid++) {
lid_gid[kk] = gid;
}
ierr = VecGetLocalSize(mpimat->lvec, &num_fine_ghosts);
CHKERRQ(ierr);
ierr = PetscMalloc1(num_fine_ghosts,&cpcol_gid);
CHKERRQ(ierr);
ierr = PetscMalloc1(num_fine_ghosts,&cpcol_state);
CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)Gmat),&sf);
CHKERRQ(ierr);
ierr = MatGetLayouts(Gmat,&layout,NULL);
CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,num_fine_ghosts,NULL,PETSC_COPY_VALUES,mpimat->garray);
CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,lid_gid,cpcol_gid);
CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,lid_gid,cpcol_gid);
CHKERRQ(ierr);
for (kk=0; kk<num_fine_ghosts; kk++) {
cpcol_state[kk]=MIS_NOT_DONE;
}
} else num_fine_ghosts = 0;
ierr = PetscMalloc1(nloc, &lid_cprowID);
CHKERRQ(ierr);
ierr = PetscMalloc1(nloc, &lid_removed);
CHKERRQ(ierr); /* explicit array needed */
if (strict_aggs) {
ierr = PetscMalloc1(nloc,&lid_parent_gid);
CHKERRQ(ierr);
}
ierr = PetscMalloc1(nloc,&lid_state);
CHKERRQ(ierr);
/* has ghost nodes for !strict and uses local indexing (yuck) */
ierr = PetscCDCreate(strict_aggs ? nloc : num_fine_ghosts+nloc, &agg_lists);
CHKERRQ(ierr);
if (a_locals_llist) *a_locals_llist = agg_lists;
/* need an inverse map - locals */
for (kk=0; kk<nloc; kk++) {
lid_cprowID[kk] = -1;
lid_removed[kk] = PETSC_FALSE;
if (strict_aggs) {
lid_parent_gid[kk] = -1.0;
}
lid_state[kk] = MIS_NOT_DONE;
}
/* set index into cmpressed row 'lid_cprowID' */
if (matB) {
for (ix=0; ix<matB->compressedrow.nrows; ix++) {
lid = matB->compressedrow.rindex[ix];
lid_cprowID[lid] = ix;
}
}
/* MIS */
iter = nremoved = nDone = 0;
ierr = ISGetIndices(perm, &perm_ix);
CHKERRQ(ierr);
while (nDone < nloc || PETSC_TRUE) { /* asyncronous not implemented */
iter++;
/* check all vertices */
for (kk=0; kk<nloc; kk++) {
lid = perm_ix[kk];
state = lid_state[lid];
if (lid_removed[lid]) continue;
if (state == MIS_NOT_DONE) {
/* parallel test, delete if selected ghost */
isOK = PETSC_TRUE;
if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i;
n = ii[ix+1] - ii[ix];
idx = matB->j + ii[ix];
for (j=0; j<n; j++) {
cpid = idx[j]; /* compressed row ID in B mat */
gid = cpcol_gid[cpid];
statej = cpcol_state[cpid];
if (statej == MIS_NOT_DONE && gid >= Iend) { /* should be (pe>rank), use gid as pe proxy */
isOK = PETSC_FALSE; /* can not delete */
break;
}
}
} /* parallel test */
if (isOK) { /* select or remove this vertex */
nDone++;
/* check for singleton */
ii = matA->i;
n = ii[lid+1] - ii[lid];
if (n < 2) {
/* if I have any ghost adj then not a sing */
ix = lid_cprowID[lid];
if (ix==-1 || (matB->compressedrow.i[ix+1]-matB->compressedrow.i[ix])==0) {
nremoved++;
lid_removed[lid] = PETSC_TRUE;
/* should select this because it is technically in the MIS but lets not */
continue; /* one local adj (me) and no ghost - singleton */
}
}
/* SELECTED state encoded with global index */
lid_state[lid] = lid+my0; /* needed???? */
nselected++;
if (strict_aggs) {
ierr = PetscCDAppendID(agg_lists, lid, lid+my0);
CHKERRQ(ierr);
} else {
ierr = PetscCDAppendID(agg_lists, lid, lid);
CHKERRQ(ierr);
}
/* delete local adj */
idx = matA->j + ii[lid];
for (j=0; j<n; j++) {
lidj = idx[j];
statej = lid_state[lidj];
if (statej == MIS_NOT_DONE) {
nDone++;
if (strict_aggs) {
ierr = PetscCDAppendID(agg_lists, lid, lidj+my0);
CHKERRQ(ierr);
} else {
ierr = PetscCDAppendID(agg_lists, lid, lidj);
CHKERRQ(ierr);
}
lid_state[lidj] = MIS_DELETED; /* delete this */
}
}
/* delete ghost adj of lid - deleted ghost done later for strict_aggs */
if (!strict_aggs) {
if ((ix=lid_cprowID[lid]) != -1) { /* if I have any ghost neighbors */
ii = matB->compressedrow.i;
n = ii[ix+1] - ii[ix];
idx = matB->j + ii[ix];
for (j=0; j<n; j++) {
cpid = idx[j]; /* compressed row ID in B mat */
statej = cpcol_state[cpid];
if (statej == MIS_NOT_DONE) {
ierr = PetscCDAppendID(agg_lists, lid, nloc+cpid);
CHKERRQ(ierr);
}
}
}
}
} /* selected */
} /* not done vertex */
} /* vertex loop */
/* update ghost states and count todos */
if (mpimat) {
/* scatter states, check for done */
ierr = PetscSFBcastBegin(sf,MPIU_INT,lid_state,cpcol_state);
CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,lid_state,cpcol_state);
CHKERRQ(ierr);
ii = matB->compressedrow.i;
for (ix=0; ix<matB->compressedrow.nrows; ix++) {
lid = matB->compressedrow.rindex[ix]; /* local boundary node */
state = lid_state[lid];
if (state == MIS_NOT_DONE) {
/* look at ghosts */
n = ii[ix+1] - ii[ix];
idx = matB->j + ii[ix];
for (j=0; j<n; j++) {
cpid = idx[j]; /* compressed row ID in B mat */
statej = cpcol_state[cpid];
if (MIS_IS_SELECTED(statej)) { /* lid is now deleted, do it */
nDone++;
lid_state[lid] = MIS_DELETED; /* delete this */
if (!strict_aggs) {
lidj = nloc + cpid;
ierr = PetscCDAppendID(agg_lists, lidj, lid);
CHKERRQ(ierr);
} else {
sgid = cpcol_gid[cpid];
lid_parent_gid[lid] = sgid; /* keep track of proc that I belong to */
}
break;
}
}
}
}
/* all done? */
t1 = nloc - nDone;
ierr = MPI_Allreduce(&t1, &t2, 1, MPIU_INT, MPI_SUM, comm);
CHKERRQ(ierr); /* synchronous version */
if (t2 == 0) break;
} else break; /* all done */
} /* outer parallel MIS loop */
ierr = ISRestoreIndices(perm,&perm_ix);
CHKERRQ(ierr);
ierr = PetscInfo3(Gmat,"\t removed %D of %D vertices. %D selected.\n",nremoved,nloc,nselected);
CHKERRQ(ierr);
/* tell adj who my lid_parent_gid vertices belong to - fill in agg_lists selected ghost lists */
if (strict_aggs && matB) {
/* need to copy this to free buffer -- should do this globaly */
ierr = PetscMalloc1(num_fine_ghosts, &cpcol_sel_gid);
CHKERRQ(ierr);
ierr = PetscMalloc1(num_fine_ghosts, &icpcol_gid);
CHKERRQ(ierr);
for (cpid=0; cpid<num_fine_ghosts; cpid++) icpcol_gid[cpid] = cpcol_gid[cpid];
/* get proc of deleted ghost */
ierr = PetscSFBcastBegin(sf,MPIU_INT,lid_parent_gid,cpcol_sel_gid);
CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,lid_parent_gid,cpcol_sel_gid);
CHKERRQ(ierr);
for (cpid=0; cpid<num_fine_ghosts; cpid++) {
sgid = cpcol_sel_gid[cpid];
gid = icpcol_gid[cpid];
if (sgid >= my0 && sgid < Iend) { /* I own this deleted */
slid = sgid - my0;
ierr = PetscCDAppendID(agg_lists, slid, gid);
CHKERRQ(ierr);
}
}
ierr = PetscFree(icpcol_gid);
CHKERRQ(ierr);
ierr = PetscFree(cpcol_sel_gid);
CHKERRQ(ierr);
}
if (mpimat) {
ierr = PetscSFDestroy(&sf);
CHKERRQ(ierr);
ierr = PetscFree(cpcol_gid);
CHKERRQ(ierr);
ierr = PetscFree(cpcol_state);
CHKERRQ(ierr);
}
ierr = PetscFree(lid_cprowID);
CHKERRQ(ierr);
ierr = PetscFree(lid_gid);
CHKERRQ(ierr);
ierr = PetscFree(lid_removed);
CHKERRQ(ierr);
if (strict_aggs) {
ierr = PetscFree(lid_parent_gid);
CHKERRQ(ierr);
}
ierr = PetscFree(lid_state);
CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MCJPGreatestWeight_Private(MatColoring mc,const PetscReal *weights,PetscReal *maxweights)
{
MC_JP *jp = (MC_JP*)mc->data;
PetscErrorCode ierr;
Mat G=mc->mat,dG,oG;
PetscBool isSeq,isMPI;
Mat_MPIAIJ *aij;
Mat_SeqAIJ *daij,*oaij;
PetscInt *di,*oi,*dj,*oj;
PetscSF sf=jp->sf;
PetscLayout layout;
PetscInt dn,on;
PetscInt i,j,l;
PetscReal *dwts=jp->dwts,*owts=jp->owts;
PetscInt ncols;
const PetscInt *cols;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)G,MATSEQAIJ,&isSeq);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)G,MATMPIAIJ,&isMPI);CHKERRQ(ierr);
if (!isSeq && !isMPI) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_ARG_WRONGSTATE,"MatColoringDegrees requires an MPI/SEQAIJ Matrix");
/* get the inner matrix structure */
oG = NULL;
oi = NULL;
oj = NULL;
if (isMPI) {
aij = (Mat_MPIAIJ*)G->data;
dG = aij->A;
oG = aij->B;
daij = (Mat_SeqAIJ*)dG->data;
oaij = (Mat_SeqAIJ*)oG->data;
di = daij->i;
dj = daij->j;
oi = oaij->i;
oj = oaij->j;
ierr = MatGetSize(oG,&dn,&on);CHKERRQ(ierr);
if (!sf) {
ierr = PetscSFCreate(PetscObjectComm((PetscObject)mc),&sf);CHKERRQ(ierr);
ierr = MatGetLayouts(G,&layout,NULL);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,on,NULL,PETSC_COPY_VALUES,aij->garray);CHKERRQ(ierr);
jp->sf = sf;
}
} else {
dG = G;
ierr = MatGetSize(dG,NULL,&dn);CHKERRQ(ierr);
daij = (Mat_SeqAIJ*)dG->data;
di = daij->i;
dj = daij->j;
}
/* set up the distance-zero weights */
if (!dwts) {
ierr = PetscMalloc1(dn,&dwts);CHKERRQ(ierr);
jp->dwts = dwts;
if (oG) {
ierr = PetscMalloc1(on,&owts);CHKERRQ(ierr);
jp->owts = owts;
}
}
for (i=0;i<dn;i++) {
maxweights[i] = weights[i];
dwts[i] = maxweights[i];
}
/* get the off-diagonal weights */
if (oG) {
ierr = PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_REAL,dwts,owts);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_REAL,dwts,owts);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
}
/* check for the maximum out to the distance of the coloring */
for (l=0;l<mc->dist;l++) {
/* check for on-diagonal greater weights */
for (i=0;i<dn;i++) {
ncols = di[i+1]-di[i];
cols = &(dj[di[i]]);
for (j=0;j<ncols;j++) {
if (dwts[cols[j]] > maxweights[i]) maxweights[i] = dwts[cols[j]];
}
/* check for off-diagonal greater weights */
if (oG) {
ncols = oi[i+1]-oi[i];
cols = &(oj[oi[i]]);
for (j=0;j<ncols;j++) {
if (owts[cols[j]] > maxweights[i]) maxweights[i] = owts[cols[j]];
}
}
}
if (l < mc->dist-1) {
for (i=0;i<dn;i++) {
dwts[i] = maxweights[i];
}
if (oG) {
ierr = PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_REAL,dwts,owts);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_REAL,dwts,owts);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
}
}
}
PetscFunctionReturn(0);
}
PetscErrorCode MCJPMinColor_Private(MatColoring mc,ISColoringValue maxcolor,const ISColoringValue *colors,ISColoringValue *mincolors)
{
MC_JP *jp = (MC_JP*)mc->data;
PetscErrorCode ierr;
Mat G=mc->mat,dG,oG;
PetscBool isSeq,isMPI;
Mat_MPIAIJ *aij;
Mat_SeqAIJ *daij,*oaij;
PetscInt *di,*oi,*dj,*oj;
PetscSF sf=jp->sf;
PetscLayout layout;
PetscInt maskrounds,maskbase,maskradix;
PetscInt dn,on;
PetscInt i,j,l,k;
PetscInt *dmask=jp->dmask,*omask=jp->omask,*cmask=jp->cmask,curmask;
PetscInt ncols;
const PetscInt *cols;
PetscFunctionBegin;
maskradix = sizeof(PetscInt)*8;
maskrounds = 1 + maxcolor / (maskradix);
maskbase = 0;
ierr = PetscObjectTypeCompare((PetscObject)G,MATSEQAIJ,&isSeq);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)G,MATMPIAIJ,&isMPI);CHKERRQ(ierr);
if (!isSeq && !isMPI) SETERRQ(PetscObjectComm((PetscObject)G),PETSC_ERR_ARG_WRONGSTATE,"MatColoringDegrees requires an MPI/SEQAIJ Matrix");
/* get the inner matrix structure */
oG = NULL;
oi = NULL;
oj = NULL;
if (isMPI) {
aij = (Mat_MPIAIJ*)G->data;
dG = aij->A;
oG = aij->B;
daij = (Mat_SeqAIJ*)dG->data;
oaij = (Mat_SeqAIJ*)oG->data;
di = daij->i;
dj = daij->j;
oi = oaij->i;
oj = oaij->j;
ierr = MatGetSize(oG,&dn,&on);CHKERRQ(ierr);
if (!sf) {
ierr = PetscSFCreate(PetscObjectComm((PetscObject)mc),&sf);CHKERRQ(ierr);
ierr = MatGetLayouts(G,&layout,NULL);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,on,NULL,PETSC_COPY_VALUES,aij->garray);CHKERRQ(ierr);
jp->sf = sf;
}
} else {
dG = G;
ierr = MatGetSize(dG,NULL,&dn);CHKERRQ(ierr);
daij = (Mat_SeqAIJ*)dG->data;
di = daij->i;
dj = daij->j;
}
for (i=0;i<dn;i++) {
mincolors[i] = IS_COLORING_MAX;
}
/* set up the distance-zero mask */
if (!dmask) {
ierr = PetscMalloc1(dn,&dmask);CHKERRQ(ierr);
ierr = PetscMalloc1(dn,&cmask);CHKERRQ(ierr);
jp->cmask = cmask;
jp->dmask = dmask;
if (oG) {
ierr = PetscMalloc1(on,&omask);CHKERRQ(ierr);
jp->omask = omask;
}
}
/* the number of colors may be more than the number of bits in a PetscInt; take multiple rounds */
for (k=0;k<maskrounds;k++) {
for (i=0;i<dn;i++) {
cmask[i] = 0;
if (colors[i] < maskbase+maskradix && colors[i] >= maskbase)
cmask[i] = 1 << (colors[i]-maskbase);
dmask[i] = cmask[i];
}
if (oG) {
ierr = PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,dmask,omask);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,dmask,omask);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
}
/* fill in the mask out to the distance of the coloring */
for (l=0;l<mc->dist;l++) {
/* fill in the on-and-off diagonal mask */
for (i=0;i<dn;i++) {
ncols = di[i+1]-di[i];
cols = &(dj[di[i]]);
for (j=0;j<ncols;j++) {
cmask[i] = cmask[i] | dmask[cols[j]];
}
if (oG) {
ncols = oi[i+1]-oi[i];
cols = &(oj[oi[i]]);
for (j=0;j<ncols;j++) {
cmask[i] = cmask[i] | omask[cols[j]];
}
}
}
for (i=0;i<dn;i++) {
dmask[i]=cmask[i];
}
if (l < mc->dist-1) {
if (oG) {
ierr = PetscLogEventBegin(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,dmask,omask);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,dmask,omask);CHKERRQ(ierr);
ierr = PetscLogEventEnd(Mat_Coloring_Comm,mc,0,0,0);CHKERRQ(ierr);
}
}
}
/* read through the mask to see if we've discovered an acceptable color for any vertices in this round */
for (i=0;i<dn;i++) {
if (mincolors[i] == IS_COLORING_MAX) {
curmask = dmask[i];
for (j=0;j<maskradix;j++) {
if (curmask % 2 == 0) {
mincolors[i] = j+maskbase;
break;
}
curmask = curmask >> 1;
}
}
}
/* do the next maskradix colors */
maskbase += maskradix;
}
static PetscErrorCode GreedyColoringLocalDistanceTwo_Private(MatColoring mc,PetscReal *wts,PetscInt *lperm,ISColoringValue *colors)
{
MC_Greedy *gr = (MC_Greedy *) mc->data;
PetscInt i,j,k,l,s,e,n,nd,nd_global,n_global,idx,ncols,maxcolors,mcol,mcol_global,nd1cols,*mask,masksize,*d1cols,*bad,*badnext,nbad,badsize,ccol,no,cbad;
Mat m = mc->mat, mt;
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)m->data;
Mat md=NULL,mo=NULL;
const PetscInt *md_i,*mo_i,*md_j,*mo_j;
const PetscInt *rmd_i,*rmo_i,*rmd_j,*rmo_j;
PetscBool isMPIAIJ,isSEQAIJ;
PetscInt pcol,*dcolors,*ocolors;
ISColoringValue *badidx;
const PetscInt *cidx;
PetscReal *owts,*colorweights;
PetscInt *oconf,*conf;
PetscSF sf;
PetscLayout layout;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = MatGetSize(m,&n_global,NULL);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(m,&s,&e);CHKERRQ(ierr);
n=e-s;
nd_global = 0;
/* get the matrix communication structures */
ierr = PetscObjectTypeCompare((PetscObject)m, MATMPIAIJ, &isMPIAIJ); CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)m, MATSEQAIJ, &isSEQAIJ); CHKERRQ(ierr);
if (isMPIAIJ) {
Mat_SeqAIJ *dseq;
Mat_SeqAIJ *oseq;
md=aij->A;
dseq = (Mat_SeqAIJ*)md->data;
mo=aij->B;
oseq = (Mat_SeqAIJ*)mo->data;
md_i = dseq->i;
md_j = dseq->j;
mo_i = oseq->i;
mo_j = oseq->j;
rmd_i = dseq->i;
rmd_j = dseq->j;
rmo_i = oseq->i;
rmo_j = oseq->j;
} else if (isSEQAIJ) {
Mat_SeqAIJ *dseq;
/* no off-processor nodes */
md=m;
dseq = (Mat_SeqAIJ*)md->data;
md_i = dseq->i;
md_j = dseq->j;
mo_i = NULL;
mo_j = NULL;
rmd_i = dseq->i;
rmd_j = dseq->j;
rmo_i = NULL;
rmo_j = NULL;
} else SETERRQ(PetscObjectComm((PetscObject)mc),PETSC_ERR_ARG_WRONG,"Matrix must be AIJ for greedy coloring");
if (!gr->symmetric) {
ierr = MatTranspose(m, MAT_INITIAL_MATRIX, &mt);CHKERRQ(ierr);
if (isSEQAIJ) {
Mat_SeqAIJ *dseq = (Mat_SeqAIJ*) mt->data;
rmd_i = dseq->i;
rmd_j = dseq->j;
rmo_i = NULL;
rmo_j = NULL;
} else SETERRQ(PetscObjectComm((PetscObject) mc), PETSC_ERR_SUP, "Nonsymmetric greedy coloring only works in serial");
}
/* create the vectors and communication structures if necessary */
no=0;
if (mo) {
ierr = VecGetLocalSize(aij->lvec,&no);CHKERRQ(ierr);
ierr = PetscSFCreate(PetscObjectComm((PetscObject)m),&sf);CHKERRQ(ierr);
ierr = MatGetLayouts(m,&layout,NULL);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,no,NULL,PETSC_COPY_VALUES,aij->garray);CHKERRQ(ierr);
}
ierr = MatColoringGetMaxColors(mc,&maxcolors);CHKERRQ(ierr);
masksize=n;
nbad=0;
badsize=n;
ierr = PetscMalloc1(masksize,&mask);CHKERRQ(ierr);
ierr = PetscMalloc4(n,&d1cols,n,&dcolors,n,&conf,n,&bad);CHKERRQ(ierr);
ierr = PetscMalloc2(badsize,&badidx,badsize,&badnext);CHKERRQ(ierr);
for(i=0;i<masksize;i++) {
mask[i]=-1;
}
for (i=0;i<n;i++) {
dcolors[i]=maxcolors;
bad[i]=-1;
}
for (i=0;i<badsize;i++) {
badnext[i]=-1;
}
if (mo) {
ierr = PetscMalloc3(no,&owts,no,&oconf,no,&ocolors);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_REAL,wts,owts);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_REAL,wts,owts);CHKERRQ(ierr);
for (i=0;i<no;i++) {
ocolors[i]=maxcolors;
}
} else { /* Appease overzealous -Wmaybe-initialized */
owts = NULL;
oconf = NULL;
ocolors = NULL;
}
mcol=0;
while (nd_global < n_global) {
nd=n;
/* assign lowest possible color to each local vertex */
mcol_global=0;
ierr = PetscLogEventBegin(MATCOLORING_Local,mc,0,0,0);CHKERRQ(ierr);
for (i=0;i<n;i++) {
idx=lperm[i];
if (dcolors[idx] == maxcolors) {
/* entries in bad */
cbad=bad[idx];
while (cbad>=0) {
ccol=badidx[cbad];
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
cbad=badnext[cbad];
}
/* diagonal distance-one rows */
nd1cols=0;
ncols = rmd_i[idx+1]-rmd_i[idx];
cidx = &(rmd_j[rmd_i[idx]]);
for (j=0;j<ncols;j++) {
d1cols[nd1cols] = cidx[j];
nd1cols++;
ccol=dcolors[cidx[j]];
if (ccol != maxcolors) {
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
/* off-diagonal distance-one rows */
if (mo) {
ncols = rmo_i[idx+1]-rmo_i[idx];
cidx = &(rmo_j[rmo_i[idx]]);
for (j=0;j<ncols;j++) {
ccol=ocolors[cidx[j]];
if (ccol != maxcolors) {
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
}
/* diagonal distance-two rows */
for (j=0;j<nd1cols;j++) {
ncols = md_i[d1cols[j]+1]-md_i[d1cols[j]];
cidx = &(md_j[md_i[d1cols[j]]]);
for (l=0;l<ncols;l++) {
ccol=dcolors[cidx[l]];
if (ccol != maxcolors) {
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
}
/* off-diagonal distance-two rows */
if (mo) {
for (j=0;j<nd1cols;j++) {
ncols = mo_i[d1cols[j]+1]-mo_i[d1cols[j]];
cidx = &(mo_j[mo_i[d1cols[j]]]);
for (l=0;l<ncols;l++) {
ccol=ocolors[cidx[l]];
if (ccol != maxcolors) {
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
}
}
/* assign this one the lowest color possible by seeing if there's a gap in the sequence of sorted neighbor colors */
for (j=0;j<masksize;j++) {
if (mask[j]!=idx) {
break;
}
}
pcol=j;
if (pcol>maxcolors) pcol=maxcolors;
dcolors[idx]=pcol;
if (pcol>mcol) mcol=pcol;
}
}
ierr = PetscLogEventEnd(MATCOLORING_Local,mc,0,0,0);CHKERRQ(ierr);
if (mo) {
/* transfer neighbor colors */
ierr = PetscSFBcastBegin(sf,MPIU_INT,dcolors,ocolors);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,dcolors,ocolors);CHKERRQ(ierr);
/* find the maximum color assigned locally and allocate a mask */
ierr = MPIU_Allreduce(&mcol,&mcol_global,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)mc));CHKERRQ(ierr);
ierr = PetscMalloc1(mcol_global+1,&colorweights);CHKERRQ(ierr);
/* check for conflicts */
for (i=0;i<n;i++) {
conf[i]=PETSC_FALSE;
}
for (i=0;i<no;i++) {
oconf[i]=PETSC_FALSE;
}
for (i=0;i<n;i++) {
ncols = mo_i[i+1]-mo_i[i];
cidx = &(mo_j[mo_i[i]]);
if (ncols > 0) {
/* fill in the mask */
for (j=0;j<mcol_global+1;j++) {
colorweights[j]=0;
}
colorweights[dcolors[i]]=wts[i];
/* fill in the off-diagonal part of the mask */
for (j=0;j<ncols;j++) {
ccol=ocolors[cidx[j]];
if (ccol < maxcolors) {
if (colorweights[ccol] < owts[cidx[j]]) {
colorweights[ccol] = owts[cidx[j]];
}
}
}
/* fill in the on-diagonal part of the mask */
ncols = md_i[i+1]-md_i[i];
cidx = &(md_j[md_i[i]]);
for (j=0;j<ncols;j++) {
ccol=dcolors[cidx[j]];
if (ccol < maxcolors) {
if (colorweights[ccol] < wts[cidx[j]]) {
colorweights[ccol] = wts[cidx[j]];
}
}
}
/* go back through and set up on and off-diagonal conflict vectors */
ncols = md_i[i+1]-md_i[i];
cidx = &(md_j[md_i[i]]);
for (j=0;j<ncols;j++) {
ccol=dcolors[cidx[j]];
if (ccol < maxcolors) {
if (colorweights[ccol] > wts[cidx[j]]) {
conf[cidx[j]]=PETSC_TRUE;
}
}
}
ncols = mo_i[i+1]-mo_i[i];
cidx = &(mo_j[mo_i[i]]);
for (j=0;j<ncols;j++) {
ccol=ocolors[cidx[j]];
if (ccol < maxcolors) {
if (colorweights[ccol] > owts[cidx[j]]) {
oconf[cidx[j]]=PETSC_TRUE;
}
}
}
}
}
nd_global=0;
ierr = PetscFree(colorweights);CHKERRQ(ierr);
ierr = PetscLogEventBegin(MATCOLORING_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFReduceBegin(sf,MPIU_INT,oconf,conf,MPIU_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sf,MPIU_INT,oconf,conf,MPIU_SUM);CHKERRQ(ierr);
ierr = PetscLogEventEnd(MATCOLORING_Comm,mc,0,0,0);CHKERRQ(ierr);
/* go through and unset local colors that have conflicts */
for (i=0;i<n;i++) {
if (conf[i]>0) {
/* push this color onto the bad stack */
badidx[nbad]=dcolors[i];
badnext[nbad]=bad[i];
bad[i]=nbad;
nbad++;
if (nbad>=badsize) {
PetscInt *newbadnext;
ISColoringValue *newbadidx;
ierr = PetscMalloc2(badsize*2,&newbadidx,badsize*2,&newbadnext);CHKERRQ(ierr);
for(k=0;k<2*badsize;k++) {
newbadnext[k]=-1;
}
for(k=0;k<badsize;k++) {
newbadidx[k]=badidx[k];
newbadnext[k]=badnext[k];
}
ierr = PetscFree2(badidx,badnext);CHKERRQ(ierr);
badidx=newbadidx;
badnext=newbadnext;
badsize*=2;
}
dcolors[i] = maxcolors;
nd--;
}
}
}
ierr = MPIU_Allreduce(&nd,&nd_global,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)mc));CHKERRQ(ierr);
}
if (mo) {
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = PetscFree3(owts,oconf,ocolors);CHKERRQ(ierr);
}
for (i=0;i<n;i++) {
colors[i]=dcolors[i];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
ierr = PetscFree4(d1cols,dcolors,conf,bad);CHKERRQ(ierr);
ierr = PetscFree2(badidx,badnext);CHKERRQ(ierr);
if (!gr->symmetric) {ierr = MatDestroy(&mt);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
static PetscErrorCode GreedyColoringLocalDistanceOne_Private(MatColoring mc,PetscReal *wts,PetscInt *lperm,ISColoringValue *colors)
{
PetscInt i,j,k,s,e,n,no,nd,nd_global,n_global,idx,ncols,maxcolors,masksize,ccol,*mask;
PetscErrorCode ierr;
Mat m=mc->mat;
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)m->data;
Mat md=NULL,mo=NULL;
const PetscInt *md_i,*mo_i,*md_j,*mo_j;
PetscBool isMPIAIJ,isSEQAIJ;
ISColoringValue pcol;
const PetscInt *cidx;
PetscInt *lcolors,*ocolors;
PetscReal *owts=NULL;
PetscSF sf;
PetscLayout layout;
PetscFunctionBegin;
ierr = MatGetSize(m,&n_global,NULL);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(m,&s,&e);CHKERRQ(ierr);
n=e-s;
masksize=20;
nd_global = 0;
/* get the matrix communication structures */
ierr = PetscObjectTypeCompare((PetscObject)m, MATMPIAIJ, &isMPIAIJ); CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)m, MATSEQAIJ, &isSEQAIJ); CHKERRQ(ierr);
if (isMPIAIJ) {
/* get the CSR data for on and off diagonal portions of m */
Mat_SeqAIJ *dseq;
Mat_SeqAIJ *oseq;
md=aij->A;
dseq = (Mat_SeqAIJ*)md->data;
mo=aij->B;
oseq = (Mat_SeqAIJ*)mo->data;
md_i = dseq->i;
md_j = dseq->j;
mo_i = oseq->i;
mo_j = oseq->j;
} else if (isSEQAIJ) {
/* get the CSR data for m */
Mat_SeqAIJ *dseq;
/* no off-processor nodes */
md=m;
dseq = (Mat_SeqAIJ*)md->data;
mo=NULL;
no=0;
md_i = dseq->i;
md_j = dseq->j;
mo_i = NULL;
mo_j = NULL;
} else SETERRQ(PetscObjectComm((PetscObject)mc),PETSC_ERR_ARG_WRONG,"Matrix must be AIJ for greedy coloring");
ierr = MatColoringGetMaxColors(mc,&maxcolors);CHKERRQ(ierr);
if (mo) {
ierr = VecGetSize(aij->lvec,&no);CHKERRQ(ierr);
ierr = PetscMalloc2(no,&ocolors,no,&owts);CHKERRQ(ierr);
for(i=0;i<no;i++) {
ocolors[i]=maxcolors;
}
}
ierr = PetscMalloc1(masksize,&mask);CHKERRQ(ierr);
ierr = PetscMalloc1(n,&lcolors);CHKERRQ(ierr);
for(i=0;i<n;i++) {
lcolors[i]=maxcolors;
}
for (i=0;i<masksize;i++) {
mask[i]=-1;
}
if (mo) {
/* transfer neighbor weights */
ierr = PetscSFCreate(PetscObjectComm((PetscObject)m),&sf);CHKERRQ(ierr);
ierr = MatGetLayouts(m,&layout,NULL);CHKERRQ(ierr);
ierr = PetscSFSetGraphLayout(sf,layout,no,NULL,PETSC_COPY_VALUES,aij->garray);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_REAL,wts,owts);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_REAL,wts,owts);CHKERRQ(ierr);
}
while (nd_global < n_global) {
nd=n;
/* assign lowest possible color to each local vertex */
ierr = PetscLogEventBegin(MATCOLORING_Local,mc,0,0,0);CHKERRQ(ierr);
for (i=0;i<n;i++) {
idx=lperm[i];
if (lcolors[idx] == maxcolors) {
ncols = md_i[idx+1]-md_i[idx];
cidx = &(md_j[md_i[idx]]);
for (j=0;j<ncols;j++) {
if (lcolors[cidx[j]] != maxcolors) {
ccol=lcolors[cidx[j]];
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
if (mo) {
ncols = mo_i[idx+1]-mo_i[idx];
cidx = &(mo_j[mo_i[idx]]);
for (j=0;j<ncols;j++) {
if (ocolors[cidx[j]] != maxcolors) {
ccol=ocolors[cidx[j]];
if (ccol>=masksize) {
PetscInt *newmask;
ierr = PetscMalloc1(masksize*2,&newmask);CHKERRQ(ierr);
for(k=0;k<2*masksize;k++) {
newmask[k]=-1;
}
for(k=0;k<masksize;k++) {
newmask[k]=mask[k];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
mask=newmask;
masksize*=2;
}
mask[ccol]=idx;
}
}
}
for (j=0;j<masksize;j++) {
if (mask[j]!=idx) {
break;
}
}
pcol=j;
if (pcol>maxcolors)pcol=maxcolors;
lcolors[idx]=pcol;
}
}
ierr = PetscLogEventEnd(MATCOLORING_Local,mc,0,0,0);CHKERRQ(ierr);
if (mo) {
/* transfer neighbor colors */
ierr = PetscLogEventBegin(MATCOLORING_Comm,mc,0,0,0);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,lcolors,ocolors);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,lcolors,ocolors);CHKERRQ(ierr);
/* check for conflicts -- this is merely checking if any adjacent off-processor rows have the same color and marking the ones that are lower weight locally for changing */
for (i=0;i<n;i++) {
ncols = mo_i[i+1]-mo_i[i];
cidx = &(mo_j[mo_i[i]]);
for (j=0;j<ncols;j++) {
/* in the case of conflicts, the highest weight one stays and the others go */
if ((ocolors[cidx[j]] == lcolors[i]) && (owts[cidx[j]] > wts[i]) && lcolors[i] < maxcolors) {
lcolors[i]=maxcolors;
nd--;
}
}
}
nd_global=0;
}
ierr = MPIU_Allreduce(&nd,&nd_global,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)mc));CHKERRQ(ierr);
}
for (i=0;i<n;i++) {
colors[i] = (ISColoringValue)lcolors[i];
}
ierr = PetscFree(mask);CHKERRQ(ierr);
ierr = PetscFree(lcolors);CHKERRQ(ierr);
if (mo) {
ierr = PetscFree2(ocolors,owts);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
/*
* The interface should be easy to use for both MatGetSubMatrix (parallel sub-matrix) and MatGetSubMatrices (sequential sub-matrices)
* */
static PetscErrorCode MatGetSubMatrix_MPIAdj_data(Mat adj,IS irows, IS icols, PetscInt **sadj_xadj,PetscInt **sadj_adjncy,PetscInt **sadj_values)
{
PetscInt nlrows_is,icols_n,i,j,nroots,nleaves,owner,rlocalindex,*ncols_send,*ncols_recv;
PetscInt nlrows_mat,*adjncy_recv,Ncols_recv,Ncols_send,*xadj_recv,*values_recv;
PetscInt *ncols_recv_offsets,loc,rnclos,*sadjncy,*sxadj,*svalues,isvalue;
const PetscInt *irows_indices,*icols_indices,*xadj, *adjncy;
Mat_MPIAdj *a = (Mat_MPIAdj*)adj->data;
PetscLayout rmap;
MPI_Comm comm;
PetscSF sf;
PetscSFNode *iremote;
PetscBool done;
PetscErrorCode ierr;
PetscFunctionBegin;
/* communicator */
ierr = PetscObjectGetComm((PetscObject)adj,&comm);CHKERRQ(ierr);
/* Layouts */
ierr = MatGetLayouts(adj,&rmap,PETSC_NULL);CHKERRQ(ierr);
/* get rows information */
ierr = ISGetLocalSize(irows,&nlrows_is);CHKERRQ(ierr);
ierr = ISGetIndices(irows,&irows_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(nlrows_is,&iremote);CHKERRQ(ierr);
/* construct sf graph*/
nleaves = nlrows_is;
for(i=0; i<nlrows_is; i++){
owner = -1;
rlocalindex = -1;
ierr = PetscLayoutFindOwnerIndex(rmap,irows_indices[i],&owner,&rlocalindex);CHKERRQ(ierr);
iremote[i].rank = owner;
iremote[i].index = rlocalindex;
}
ierr = MatGetRowIJ(adj,0,PETSC_FALSE,PETSC_FALSE,&nlrows_mat,&xadj,&adjncy,&done);CHKERRQ(ierr);
ierr = PetscCalloc4(nlrows_mat,&ncols_send,nlrows_is,&xadj_recv,nlrows_is+1,&ncols_recv_offsets,nlrows_is,&ncols_recv);CHKERRQ(ierr);
nroots = nlrows_mat;
for(i=0; i<nlrows_mat; i++){
ncols_send[i] = xadj[i+1]-xadj[i];
}
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,ncols_send,ncols_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,ncols_send,ncols_recv);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,xadj,xadj_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,xadj,xadj_recv);CHKERRQ(ierr);
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
Ncols_recv =0;
for(i=0; i<nlrows_is; i++){
Ncols_recv += ncols_recv[i];
ncols_recv_offsets[i+1] = ncols_recv[i]+ncols_recv_offsets[i];
}
Ncols_send = 0;
for(i=0; i<nlrows_mat; i++){
Ncols_send += ncols_send[i];
}
ierr = PetscCalloc1(Ncols_recv,&iremote);CHKERRQ(ierr);
ierr = PetscCalloc1(Ncols_recv,&adjncy_recv);CHKERRQ(ierr);
nleaves = Ncols_recv;
Ncols_recv = 0;
for(i=0; i<nlrows_is; i++){
ierr = PetscLayoutFindOwner(rmap,irows_indices[i],&owner);CHKERRQ(ierr);
for(j=0; j<ncols_recv[i]; j++){
iremote[Ncols_recv].rank = owner;
iremote[Ncols_recv++].index = xadj_recv[i]+j;
}
}
ierr = ISRestoreIndices(irows,&irows_indices);CHKERRQ(ierr);
/*if we need to deal with edge weights ???*/
if(a->values){isvalue=1;}else{isvalue=0;}
/*involve a global communication */
/*ierr = MPI_Allreduce(&isvalue,&isvalue,1,MPIU_INT,MPI_SUM,comm);CHKERRQ(ierr);*/
if(isvalue){ierr = PetscCalloc1(Ncols_recv,&values_recv);CHKERRQ(ierr);}
nroots = Ncols_send;
ierr = PetscSFCreate(comm,&sf);CHKERRQ(ierr);
ierr = PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);CHKERRQ(ierr);
ierr = PetscSFSetType(sf,PETSCSFBASIC);CHKERRQ(ierr);
ierr = PetscSFSetFromOptions(sf);CHKERRQ(ierr);
ierr = PetscSFBcastBegin(sf,MPIU_INT,adjncy,adjncy_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,adjncy,adjncy_recv);CHKERRQ(ierr);
if(isvalue){
ierr = PetscSFBcastBegin(sf,MPIU_INT,a->values,values_recv);CHKERRQ(ierr);
ierr = PetscSFBcastEnd(sf,MPIU_INT,a->values,values_recv);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sf);CHKERRQ(ierr);
ierr = MatRestoreRowIJ(adj,0,PETSC_FALSE,PETSC_FALSE,&nlrows_mat,&xadj,&adjncy,&done);CHKERRQ(ierr);
ierr = ISGetLocalSize(icols,&icols_n);CHKERRQ(ierr);
ierr = ISGetIndices(icols,&icols_indices);CHKERRQ(ierr);
rnclos = 0;
for(i=0; i<nlrows_is; i++){
for(j=ncols_recv_offsets[i]; j<ncols_recv_offsets[i+1]; j++){
ierr = PetscFindInt(adjncy_recv[j], icols_n, icols_indices, &loc);CHKERRQ(ierr);
if(loc<0){
adjncy_recv[j] = -1;
if(isvalue) values_recv[j] = -1;
ncols_recv[i]--;
}else{
rnclos++;
}
}
}
ierr = ISRestoreIndices(icols,&icols_indices);CHKERRQ(ierr);
ierr = PetscCalloc1(rnclos,&sadjncy);CHKERRQ(ierr);
if(isvalue) {ierr = PetscCalloc1(rnclos,&svalues);CHKERRQ(ierr);}
ierr = PetscCalloc1(nlrows_is+1,&sxadj);CHKERRQ(ierr);
rnclos = 0;
for(i=0; i<nlrows_is; i++){
for(j=ncols_recv_offsets[i]; j<ncols_recv_offsets[i+1]; j++){
if(adjncy_recv[j]<0) continue;
sadjncy[rnclos] = adjncy_recv[j];
if(isvalue) svalues[rnclos] = values_recv[j];
rnclos++;
}
}
for(i=0; i<nlrows_is; i++){
sxadj[i+1] = sxadj[i]+ncols_recv[i];
}
if(sadj_xadj) { *sadj_xadj = sxadj;}else { ierr = PetscFree(sxadj);CHKERRQ(ierr);}
if(sadj_adjncy){ *sadj_adjncy = sadjncy;}else{ ierr = PetscFree(sadjncy);CHKERRQ(ierr);}
if(sadj_values){
if(isvalue) *sadj_values = svalues; else *sadj_values=0;
}else{
if(isvalue) {ierr = PetscFree(svalues);CHKERRQ(ierr);}
}
ierr = PetscFree4(ncols_send,xadj_recv,ncols_recv_offsets,ncols_recv);CHKERRQ(ierr);
ierr = PetscFree(adjncy_recv);CHKERRQ(ierr);
if(isvalue) {ierr = PetscFree(values_recv);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
