/*@
ISComplement - Given an index set (IS) generates the complement index set. That is all
all indices that are NOT in the given set.
Collective on IS
Input Parameter:
+ is - the index set
. nmin - the first index desired in the local part of the complement
- nmax - the largest index desired in the local part of the complement (note that all indices in is must be greater or equal to nmin and less than nmax)
Output Parameter:
. isout - the complement
Notes: The communicator for this new IS is the same as for the input IS
For a parallel IS, this will generate the local part of the complement on each process
To generate the entire complement (on each process) of a parallel IS, first call ISAllGather() and then
call this routine.
Level: intermediate
Concepts: gather^index sets
Concepts: index sets^gathering to all processors
Concepts: IS^gathering to all processors
.seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather()
@*/
PetscErrorCode ISComplement(IS is,PetscInt nmin,PetscInt nmax,IS *isout)
{
PetscErrorCode ierr;
const PetscInt *indices;
PetscInt n,i,j,unique,cnt,*nindices;
PetscBool sorted;
PetscFunctionBegin;
PetscValidHeaderSpecific(is,IS_CLASSID,1);
PetscValidPointer(isout,3);
if (nmin < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"nmin %D cannot be negative",nmin);
if (nmin > nmax) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"nmin %D cannot be greater than nmax %D",nmin,nmax);
ierr = ISSorted(is,&sorted);CHKERRQ(ierr);
if (!sorted) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Index set must be sorted");
ierr = ISGetLocalSize(is,&n);CHKERRQ(ierr);
ierr = ISGetIndices(is,&indices);CHKERRQ(ierr);
#if defined(PETSC_USE_DEBUG)
for (i=0; i<n; i++) {
if (indices[i] < nmin) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Index %D's value %D is smaller than minimum given %D",i,indices[i],nmin);
if (indices[i] >= nmax) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Index %D's value %D is larger than maximum given %D",i,indices[i],nmax);
}
#endif
/* Count number of unique entries */
unique = (n>0);
for (i=0; i<n-1; i++) {
if (indices[i+1] != indices[i]) unique++;
}
ierr = PetscMalloc1(nmax-nmin-unique,&nindices);CHKERRQ(ierr);
cnt = 0;
for (i=nmin,j=0; i<nmax; i++) {
if (j<n && i==indices[j]) do { j++; } while (j<n && i==indices[j]);
else nindices[cnt++] = i;
}
if (cnt != nmax-nmin-unique) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Number of entries found in complement %D does not match expected %D",cnt,nmax-nmin-unique);
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)is),cnt,nindices,PETSC_OWN_POINTER,isout);CHKERRQ(ierr);
ierr = ISRestoreIndices(is,&indices);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
PetscMPIInt rank,size;
PetscInt i,n,*indices;
const PetscInt *ii;
IS is,newis;
PetscBool flg;
PetscErrorCode ierr;
ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
/*
Test IS of size 0
*/
ierr = ISCreateGeneral(PETSC_COMM_SELF,0,&n,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = ISGetSize(is,&n);CHKERRQ(ierr);
if (n != 0) SETERRQ(PETSC_COMM_SELF,1,"ISGetSize");
ierr = ISDestroy(&is);CHKERRQ(ierr);
/*
Create large IS and test ISGetIndices()
*/
n = 10000 + rank;
ierr = PetscMalloc1(n,&indices);CHKERRQ(ierr);
for (i=0; i<n; i++) indices[i] = rank + i;
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = ISGetIndices(is,&ii);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (ii[i] != indices[i]) SETERRQ(PETSC_COMM_SELF,1,"ISGetIndices");
}
ierr = ISRestoreIndices(is,&ii);CHKERRQ(ierr);
/*
Check identity and permutation
*/
ierr = ISPermutation(is,&flg);CHKERRQ(ierr);
if (flg) SETERRQ(PETSC_COMM_SELF,1,"ISPermutation");
ierr = ISIdentity(is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISIdentity");
ierr = ISSetPermutation(is);CHKERRQ(ierr);
ierr = ISSetIdentity(is);CHKERRQ(ierr);
ierr = ISPermutation(is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISPermutation");
ierr = ISIdentity(is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISIdentity");
/*
Check equality of index sets
*/
ierr = ISEqual(is,is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISEqual");
/*
Sorting
*/
ierr = ISSort(is);CHKERRQ(ierr);
ierr = ISSorted(is,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"ISSort");
/*
Thinks it is a different type?
*/
ierr = PetscObjectTypeCompare((PetscObject)is,ISSTRIDE,&flg);CHKERRQ(ierr);
if (flg) SETERRQ(PETSC_COMM_SELF,1,"ISStride");
ierr = PetscObjectTypeCompare((PetscObject)is,ISBLOCK,&flg);CHKERRQ(ierr);
if (flg) SETERRQ(PETSC_COMM_SELF,1,"ISBlock");
ierr = ISDestroy(&is);CHKERRQ(ierr);
/*
Inverting permutation
*/
for (i=0; i<n; i++) indices[i] = n - i - 1;
ierr = ISCreateGeneral(PETSC_COMM_SELF,n,indices,PETSC_COPY_VALUES,&is);CHKERRQ(ierr);
ierr = PetscFree(indices);CHKERRQ(ierr);
ierr = ISSetPermutation(is);CHKERRQ(ierr);
ierr = ISInvertPermutation(is,PETSC_DECIDE,&newis);CHKERRQ(ierr);
ierr = ISGetIndices(newis,&ii);CHKERRQ(ierr);
for (i=0; i<n; i++) {
if (ii[i] != n - i - 1) SETERRQ(PETSC_COMM_SELF,1,"ISInvertPermutation");
}
ierr = ISRestoreIndices(newis,&ii);CHKERRQ(ierr);
ierr = ISDestroy(&newis);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*@
ISSum - Computes the sum (union) of two index sets.
Only sequential version (at the moment)
Input Parameter:
+ is1 - index set to be extended
- is2 - index values to be added
Output Parameter:
. is3 - the sum; this can not be is1 or is2
Notes:
If n1 and n2 are the sizes of the sets, this takes O(n1+n2) time;
Both index sets need to be sorted on input.
Level: intermediate
.seealso: ISDestroy(), ISView(), ISDifference(), ISExpand()
Concepts: index sets^union
Concepts: IS^union
@*/
PetscErrorCode ISSum(IS is1,IS is2,IS *is3)
{
MPI_Comm comm;
PetscBool f;
PetscMPIInt size;
const PetscInt *i1,*i2;
PetscInt n1,n2,n3, p1,p2, *iout;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(is1,IS_CLASSID,1);
PetscValidHeaderSpecific(is2,IS_CLASSID,2);
ierr = PetscObjectGetComm((PetscObject)(is1),&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size>1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Currently only for uni-processor IS");
ierr = ISSorted(is1,&f);CHKERRQ(ierr);
if (!f) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Arg 1 is not sorted");
ierr = ISSorted(is2,&f);CHKERRQ(ierr);
if (!f) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Arg 2 is not sorted");
ierr = ISGetLocalSize(is1,&n1);CHKERRQ(ierr);
ierr = ISGetLocalSize(is2,&n2);CHKERRQ(ierr);
if (!n2) PetscFunctionReturn(0);
ierr = ISGetIndices(is1,&i1);CHKERRQ(ierr);
ierr = ISGetIndices(is2,&i2);CHKERRQ(ierr);
p1 = 0; p2 = 0; n3 = 0;
do {
if (p1==n1) { /* cleanup for is2 */ n3 += n2-p2; break;
} else {
while (p2<n2 && i2[p2]<i1[p1]) {
n3++; p2++;
}
if (p2==n2) {
/* cleanup for is1 */
n3 += n1-p1; break;
} else {
if (i2[p2]==i1[p1]) { n3++; p1++; p2++; }
}
}
if (p2==n2) {
/* cleanup for is1 */
n3 += n1-p1; break;
} else {
while (p1<n1 && i1[p1]<i2[p2]) {
n3++; p1++;
}
if (p1==n1) {
/* clean up for is2 */
n3 += n2-p2; break;
} else {
if (i1[p1]==i2[p2]) { n3++; p1++; p2++; }
}
}
} while (p1<n1 || p2<n2);
if (n3==n1) { /* no new elements to be added */
ierr = ISRestoreIndices(is1,&i1);CHKERRQ(ierr);
ierr = ISRestoreIndices(is2,&i2);CHKERRQ(ierr);
ierr = ISDuplicate(is1,is3);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
ierr = PetscMalloc1(n3,&iout);CHKERRQ(ierr);
p1 = 0; p2 = 0; n3 = 0;
do {
if (p1==n1) { /* cleanup for is2 */
while (p2<n2) iout[n3++] = i2[p2++];
break;
} else {
while (p2<n2 && i2[p2]<i1[p1]) iout[n3++] = i2[p2++];
if (p2==n2) { /* cleanup for is1 */
while (p1<n1) iout[n3++] = i1[p1++];
break;
} else {
if (i2[p2]==i1[p1]) { iout[n3++] = i1[p1++]; p2++; }
}
}
if (p2==n2) { /* cleanup for is1 */
while (p1<n1) iout[n3++] = i1[p1++];
break;
} else {
while (p1<n1 && i1[p1]<i2[p2]) iout[n3++] = i1[p1++];
if (p1==n1) { /* clean up for is2 */
while (p2<n2) iout[n3++] = i2[p2++];
break;
} else {
if (i1[p1]==i2[p2]) { iout[n3++] = i1[p1++]; p2++; }
}
}
} while (p1<n1 || p2<n2);
ierr = ISRestoreIndices(is1,&i1);CHKERRQ(ierr);
ierr = ISRestoreIndices(is2,&i2);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,n3,iout,PETSC_OWN_POINTER,is3);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode ISCompressIndicesSorted(PetscInt n,PetscInt bs,PetscInt imax,const IS is_in[],IS is_out[])
{
PetscErrorCode ierr;
PetscInt i,j,k,val,len,*nidx,bbs;
const PetscInt *idx,*idx_local;
PetscBool flg,isblock;
#if defined(PETSC_USE_CTABLE)
PetscInt maxsz;
#else
PetscInt Nbs=n/bs;
#endif
PetscFunctionBegin;
for (i=0; i<imax; i++) {
ierr = ISSorted(is_in[i],&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Indices are not sorted");
}
#if defined(PETSC_USE_CTABLE)
/* Now check max size */
for (i=0,maxsz=0; i<imax; i++) {
ierr = ISGetLocalSize(is_in[i],&len);CHKERRQ(ierr);
if (len%bs !=0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Indices are not block ordered");
len = len/bs; /* The reduced index size */
if (len > maxsz) maxsz = len;
}
ierr = PetscMalloc1(maxsz,&nidx);CHKERRQ(ierr);
#else
ierr = PetscMalloc1(Nbs,&nidx);CHKERRQ(ierr);
#endif
/* Now check if the indices are in block order */
for (i=0; i<imax; i++) {
ierr = ISGetLocalSize(is_in[i],&len);CHKERRQ(ierr);
/* special case where IS is already block IS of the correct size */
ierr = PetscObjectTypeCompare((PetscObject)is_in[i],ISBLOCK,&isblock);CHKERRQ(ierr);
if (isblock) {
ierr = ISBlockGetLocalSize(is_in[i],&bbs);CHKERRQ(ierr);
if (bs == bbs) {
len = len/bs;
ierr = ISBlockGetIndices(is_in[i],&idx);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,len,idx,PETSC_COPY_VALUES,(is_out+i));CHKERRQ(ierr);
ierr = ISBlockRestoreIndices(is_in[i],&idx);CHKERRQ(ierr);
continue;
}
}
ierr = ISGetIndices(is_in[i],&idx);CHKERRQ(ierr);
if (len%bs !=0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Indices are not block ordered");
len = len/bs; /* The reduced index size */
idx_local = idx;
for (j=0; j<len; j++) {
val = idx_local[0];
if (val%bs != 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Indices are not block ordered");
for (k=0; k<bs; k++) {
if (val+k != idx_local[k]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"Indices are not block ordered");
}
nidx[j] = val/bs;
idx_local += bs;
}
ierr = ISRestoreIndices(is_in[i],&idx);CHKERRQ(ierr);
ierr = ISCreateGeneral(PETSC_COMM_SELF,len,nidx,PETSC_COPY_VALUES,(is_out+i));CHKERRQ(ierr);
}
ierr = PetscFree(nidx);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatGetSubMatrices_MPIDense_Local(Mat C,PetscInt ismax,const IS isrow[],const IS iscol[],MatReuse scall,Mat *submats)
{
Mat_MPIDense *c = (Mat_MPIDense*)C->data;
Mat A = c->A;
Mat_SeqDense *a = (Mat_SeqDense*)A->data,*mat;
PetscErrorCode ierr;
PetscMPIInt rank,size,tag0,tag1,idex,end,i;
PetscInt N = C->cmap->N,rstart = C->rmap->rstart,count;
const PetscInt **irow,**icol,*irow_i;
PetscInt *nrow,*ncol,*w1,*w3,*w4,*rtable,start;
PetscInt **sbuf1,m,j,k,l,ct1,**rbuf1,row,proc;
PetscInt nrqs,msz,**ptr,*ctr,*pa,*tmp,bsz,nrqr;
PetscInt is_no,jmax,**rmap,*rmap_i;
PetscInt ctr_j,*sbuf1_j,*rbuf1_i;
MPI_Request *s_waits1,*r_waits1,*s_waits2,*r_waits2;
MPI_Status *r_status1,*r_status2,*s_status1,*s_status2;
MPI_Comm comm;
PetscScalar **rbuf2,**sbuf2;
PetscBool sorted;
PetscFunctionBegin;
comm = ((PetscObject)C)->comm;
tag0 = ((PetscObject)C)->tag;
size = c->size;
rank = c->rank;
m = C->rmap->N;
/* Get some new tags to keep the communication clean */
ierr = PetscObjectGetNewTag((PetscObject)C,&tag1);CHKERRQ(ierr);
/* Check if the col indices are sorted */
for (i=0; i<ismax; i++) {
ierr = ISSorted(isrow[i],&sorted);CHKERRQ(ierr);
if (!sorted) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"ISrow is not sorted");
ierr = ISSorted(iscol[i],&sorted);CHKERRQ(ierr);
if (!sorted) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"IScol is not sorted");
}
ierr = PetscMalloc5(ismax,const PetscInt*,&irow,ismax,const PetscInt*,&icol,ismax,PetscInt,&nrow,ismax,PetscInt,&ncol,m,PetscInt,&rtable);CHKERRQ(ierr);
for (i=0; i<ismax; i++) {
ierr = ISGetIndices(isrow[i],&irow[i]);CHKERRQ(ierr);
ierr = ISGetIndices(iscol[i],&icol[i]);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrow[i],&nrow[i]);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscol[i],&ncol[i]);CHKERRQ(ierr);
}
/* Create hash table for the mapping :row -> proc*/
for (i=0,j=0; i<size; i++) {
jmax = C->rmap->range[i+1];
for (; j<jmax; j++) {
rtable[j] = i;
}
}
/* evaluate communication - mesg to who,length of mesg, and buffer space
required. Based on this, buffers are allocated, and data copied into them*/
ierr = PetscMalloc3(2*size,PetscInt,&w1,size,PetscInt,&w3,size,PetscInt,&w4);CHKERRQ(ierr);
ierr = PetscMemzero(w1,size*2*sizeof(PetscInt));CHKERRQ(ierr); /* initialize work vector*/
ierr = PetscMemzero(w3,size*sizeof(PetscInt));CHKERRQ(ierr); /* initialize work vector*/
for (i=0; i<ismax; i++) {
ierr = PetscMemzero(w4,size*sizeof(PetscInt));CHKERRQ(ierr); /* initialize work vector*/
jmax = nrow[i];
irow_i = irow[i];
for (j=0; j<jmax; j++) {
row = irow_i[j];
proc = rtable[row];
w4[proc]++;
}
for (j=0; j<size; j++) {
if (w4[j]) { w1[2*j] += w4[j]; w3[j]++;}
}
}
nrqs = 0; /* no of outgoing messages */
msz = 0; /* total mesg length (for all procs) */
w1[2*rank] = 0; /* no mesg sent to self */
w3[rank] = 0;
for (i=0; i<size; i++) {
if (w1[2*i]) { w1[2*i+1] = 1; nrqs++;} /* there exists a message to proc i */
}
ierr = PetscMalloc((nrqs+1)*sizeof(PetscInt),&pa);CHKERRQ(ierr); /*(proc -array)*/
for (i=0,j=0; i<size; i++) {
if (w1[2*i]) { pa[j] = i; j++; }
}
/* Each message would have a header = 1 + 2*(no of IS) + data */
for (i=0; i<nrqs; i++) {
j = pa[i];
w1[2*j] += w1[2*j+1] + 2* w3[j];
msz += w1[2*j];
}
/* Do a global reduction to determine how many messages to expect*/
ierr = PetscMaxSum(comm,w1,&bsz,&nrqr);CHKERRQ(ierr);
/* Allocate memory for recv buffers . Make sure rbuf1[0] exists by adding 1 to the buffer length */
ierr = PetscMalloc((nrqr+1)*sizeof(PetscInt*),&rbuf1);CHKERRQ(ierr);
ierr = PetscMalloc(nrqr*bsz*sizeof(PetscInt),&rbuf1[0]);CHKERRQ(ierr);
for (i=1; i<nrqr; ++i) rbuf1[i] = rbuf1[i-1] + bsz;
/* Post the receives */
ierr = PetscMalloc((nrqr+1)*sizeof(MPI_Request),&r_waits1);CHKERRQ(ierr);
for (i=0; i<nrqr; ++i) {
ierr = MPI_Irecv(rbuf1[i],bsz,MPIU_INT,MPI_ANY_SOURCE,tag0,comm,r_waits1+i);CHKERRQ(ierr);
}
/* Allocate Memory for outgoing messages */
ierr = PetscMalloc4(size,PetscInt*,&sbuf1,size,PetscInt*,&ptr,2*msz,PetscInt,&tmp,size,PetscInt,&ctr);CHKERRQ(ierr);
ierr = PetscMemzero(sbuf1,size*sizeof(PetscInt*));CHKERRQ(ierr);
ierr = PetscMemzero(ptr,size*sizeof(PetscInt*));CHKERRQ(ierr);
{
PetscInt *iptr = tmp,ict = 0;
for (i=0; i<nrqs; i++) {
j = pa[i];
iptr += ict;
sbuf1[j] = iptr;
ict = w1[2*j];
}
}
/* Form the outgoing messages */
/* Initialize the header space */
for (i=0; i<nrqs; i++) {
j = pa[i];
sbuf1[j][0] = 0;
ierr = PetscMemzero(sbuf1[j]+1,2*w3[j]*sizeof(PetscInt));CHKERRQ(ierr);
ptr[j] = sbuf1[j] + 2*w3[j] + 1;
}
/* Parse the isrow and copy data into outbuf */
for (i=0; i<ismax; i++) {
ierr = PetscMemzero(ctr,size*sizeof(PetscInt));CHKERRQ(ierr);
irow_i = irow[i];
jmax = nrow[i];
for (j=0; j<jmax; j++) { /* parse the indices of each IS */
row = irow_i[j];
proc = rtable[row];
if (proc != rank) { /* copy to the outgoing buf*/
ctr[proc]++;
*ptr[proc] = row;
ptr[proc]++;
}
}
/* Update the headers for the current IS */
for (j=0; j<size; j++) { /* Can Optimise this loop too */
if ((ctr_j = ctr[j])) {
sbuf1_j = sbuf1[j];
k = ++sbuf1_j[0];
sbuf1_j[2*k] = ctr_j;
sbuf1_j[2*k-1] = i;
}
}
}
/* Now post the sends */
ierr = PetscMalloc((nrqs+1)*sizeof(MPI_Request),&s_waits1);CHKERRQ(ierr);
for (i=0; i<nrqs; ++i) {
j = pa[i];
ierr = MPI_Isend(sbuf1[j],w1[2*j],MPIU_INT,j,tag0,comm,s_waits1+i);CHKERRQ(ierr);
}
/* Post recieves to capture the row_data from other procs */
ierr = PetscMalloc((nrqs+1)*sizeof(MPI_Request),&r_waits2);CHKERRQ(ierr);
ierr = PetscMalloc((nrqs+1)*sizeof(PetscScalar*),&rbuf2);CHKERRQ(ierr);
for (i=0; i<nrqs; i++) {
j = pa[i];
count = (w1[2*j] - (2*sbuf1[j][0] + 1))*N;
ierr = PetscMalloc((count+1)*sizeof(PetscScalar),&rbuf2[i]);CHKERRQ(ierr);
ierr = MPI_Irecv(rbuf2[i],count,MPIU_SCALAR,j,tag1,comm,r_waits2+i);CHKERRQ(ierr);
}
/* Receive messages(row_nos) and then, pack and send off the rowvalues
to the correct processors */
ierr = PetscMalloc((nrqr+1)*sizeof(MPI_Request),&s_waits2);CHKERRQ(ierr);
ierr = PetscMalloc((nrqr+1)*sizeof(MPI_Status),&r_status1);CHKERRQ(ierr);
ierr = PetscMalloc((nrqr+1)*sizeof(PetscScalar*),&sbuf2);CHKERRQ(ierr);
{
PetscScalar *sbuf2_i,*v_start;
PetscInt s_proc;
for (i=0; i<nrqr; ++i) {
ierr = MPI_Waitany(nrqr,r_waits1,&idex,r_status1+i);CHKERRQ(ierr);
s_proc = r_status1[i].MPI_SOURCE; /* send processor */
rbuf1_i = rbuf1[idex]; /* Actual message from s_proc */
/* no of rows = end - start; since start is array idex[], 0idex, whel end
is length of the buffer - which is 1idex */
start = 2*rbuf1_i[0] + 1;
ierr = MPI_Get_count(r_status1+i,MPIU_INT,&end);CHKERRQ(ierr);
/* allocate memory sufficinet to hold all the row values */
ierr = PetscMalloc((end-start)*N*sizeof(PetscScalar),&sbuf2[idex]);CHKERRQ(ierr);
sbuf2_i = sbuf2[idex];
/* Now pack the data */
for (j=start; j<end; j++) {
row = rbuf1_i[j] - rstart;
v_start = a->v + row;
for (k=0; k<N; k++) {
sbuf2_i[0] = v_start[0];
sbuf2_i++; v_start += C->rmap->n;
}
}
/* Now send off the data */
ierr = MPI_Isend(sbuf2[idex],(end-start)*N,MPIU_SCALAR,s_proc,tag1,comm,s_waits2+i);CHKERRQ(ierr);
}
}
/* End Send-Recv of IS + row_numbers */
ierr = PetscFree(r_status1);CHKERRQ(ierr);
ierr = PetscFree(r_waits1);CHKERRQ(ierr);
ierr = PetscMalloc((nrqs+1)*sizeof(MPI_Status),&s_status1);CHKERRQ(ierr);
if (nrqs) {ierr = MPI_Waitall(nrqs,s_waits1,s_status1);CHKERRQ(ierr);}
ierr = PetscFree(s_status1);CHKERRQ(ierr);
ierr = PetscFree(s_waits1);CHKERRQ(ierr);
/* Create the submatrices */
if (scall == MAT_REUSE_MATRIX) {
for (i=0; i<ismax; i++) {
mat = (Mat_SeqDense *)(submats[i]->data);
if ((submats[i]->rmap->n != nrow[i]) || (submats[i]->cmap->n != ncol[i])) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Cannot reuse matrix. wrong size");
ierr = PetscMemzero(mat->v,submats[i]->rmap->n*submats[i]->cmap->n*sizeof(PetscScalar));CHKERRQ(ierr);
submats[i]->factortype = C->factortype;
}
} else {
for (i=0; i<ismax; i++) {
ierr = MatCreate(PETSC_COMM_SELF,submats+i);CHKERRQ(ierr);
ierr = MatSetSizes(submats[i],nrow[i],ncol[i],nrow[i],ncol[i]);CHKERRQ(ierr);
ierr = MatSetType(submats[i],((PetscObject)A)->type_name);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(submats[i],PETSC_NULL);CHKERRQ(ierr);
}
}
/* Assemble the matrices */
{
PetscInt col;
PetscScalar *imat_v,*mat_v,*imat_vi,*mat_vi;
for (i=0; i<ismax; i++) {
mat = (Mat_SeqDense*)submats[i]->data;
mat_v = a->v;
imat_v = mat->v;
irow_i = irow[i];
m = nrow[i];
for (j=0; j<m; j++) {
row = irow_i[j] ;
proc = rtable[row];
if (proc == rank) {
row = row - rstart;
mat_vi = mat_v + row;
imat_vi = imat_v + j;
for (k=0; k<ncol[i]; k++) {
col = icol[i][k];
imat_vi[k*m] = mat_vi[col*C->rmap->n];
}
}
}
}
}
/* Create row map-> This maps c->row to submat->row for each submat*/
/* this is a very expensive operation wrt memory usage */
ierr = PetscMalloc(ismax*sizeof(PetscInt*),&rmap);CHKERRQ(ierr);
ierr = PetscMalloc(ismax*C->rmap->N*sizeof(PetscInt),&rmap[0]);CHKERRQ(ierr);
ierr = PetscMemzero(rmap[0],ismax*C->rmap->N*sizeof(PetscInt));CHKERRQ(ierr);
for (i=1; i<ismax; i++) { rmap[i] = rmap[i-1] + C->rmap->N;}
for (i=0; i<ismax; i++) {
rmap_i = rmap[i];
irow_i = irow[i];
jmax = nrow[i];
for (j=0; j<jmax; j++) {
rmap_i[irow_i[j]] = j;
}
}
/* Now Receive the row_values and assemble the rest of the matrix */
ierr = PetscMalloc((nrqs+1)*sizeof(MPI_Status),&r_status2);CHKERRQ(ierr);
{
PetscInt is_max,tmp1,col,*sbuf1_i,is_sz;
PetscScalar *rbuf2_i,*imat_v,*imat_vi;
for (tmp1=0; tmp1<nrqs; tmp1++) { /* For each message */
ierr = MPI_Waitany(nrqs,r_waits2,&i,r_status2+tmp1);CHKERRQ(ierr);
/* Now dig out the corresponding sbuf1, which contains the IS data_structure */
sbuf1_i = sbuf1[pa[i]];
is_max = sbuf1_i[0];
ct1 = 2*is_max+1;
rbuf2_i = rbuf2[i];
for (j=1; j<=is_max; j++) { /* For each IS belonging to the message */
is_no = sbuf1_i[2*j-1];
is_sz = sbuf1_i[2*j];
mat = (Mat_SeqDense*)submats[is_no]->data;
imat_v = mat->v;
rmap_i = rmap[is_no];
m = nrow[is_no];
for (k=0; k<is_sz; k++,rbuf2_i+=N) { /* For each row */
row = sbuf1_i[ct1]; ct1++;
row = rmap_i[row];
imat_vi = imat_v + row;
for (l=0; l<ncol[is_no]; l++) { /* For each col */
col = icol[is_no][l];
imat_vi[l*m] = rbuf2_i[col];
}
}
}
}
}
/* End Send-Recv of row_values */
ierr = PetscFree(r_status2);CHKERRQ(ierr);
ierr = PetscFree(r_waits2);CHKERRQ(ierr);
ierr = PetscMalloc((nrqr+1)*sizeof(MPI_Status),&s_status2);CHKERRQ(ierr);
if (nrqr) {ierr = MPI_Waitall(nrqr,s_waits2,s_status2);CHKERRQ(ierr);}
ierr = PetscFree(s_status2);CHKERRQ(ierr);
ierr = PetscFree(s_waits2);CHKERRQ(ierr);
/* Restore the indices */
for (i=0; i<ismax; i++) {
ierr = ISRestoreIndices(isrow[i],irow+i);CHKERRQ(ierr);
ierr = ISRestoreIndices(iscol[i],icol+i);CHKERRQ(ierr);
}
/* Destroy allocated memory */
ierr = PetscFree5(irow,icol,nrow,ncol,rtable);CHKERRQ(ierr);
ierr = PetscFree3(w1,w3,w4);CHKERRQ(ierr);
ierr = PetscFree(pa);CHKERRQ(ierr);
for (i=0; i<nrqs; ++i) {
ierr = PetscFree(rbuf2[i]);CHKERRQ(ierr);
}
ierr = PetscFree(rbuf2);CHKERRQ(ierr);
ierr = PetscFree4(sbuf1,ptr,tmp,ctr);CHKERRQ(ierr);
ierr = PetscFree(rbuf1[0]);CHKERRQ(ierr);
ierr = PetscFree(rbuf1);CHKERRQ(ierr);
for (i=0; i<nrqr; ++i) {
ierr = PetscFree(sbuf2[i]);CHKERRQ(ierr);
}
ierr = PetscFree(sbuf2);CHKERRQ(ierr);
ierr = PetscFree(rmap[0]);CHKERRQ(ierr);
ierr = PetscFree(rmap);CHKERRQ(ierr);
for (i=0; i<ismax; i++) {
ierr = MatAssemblyBegin(submats[i],MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(submats[i],MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
void PETSC_STDCALL issorted_(IS is,PetscBool *flg, int *__ierr ){
*__ierr = ISSorted(
(IS)PetscToPointer((is) ),flg);
}
PetscErrorCode SNESSetUp_Multiblock(SNES snes)
{
SNES_Multiblock *mb = (SNES_Multiblock *) snes->data;
BlockDesc blocks;
PetscInt i, numBlocks;
PetscErrorCode ierr;
PetscFunctionBegin;
/* ierr = SNESDefaultGetWork(snes, 1);CHKERRQ(ierr); */
ierr = SNESMultiblockSetDefaults(snes);CHKERRQ(ierr);
numBlocks = mb->numBlocks;
blocks = mb->blocks;
/* Create ISs */
if (!mb->issetup) {
PetscInt ccsize, rstart, rend, nslots, bs;
PetscBool sorted;
mb->issetup = PETSC_TRUE;
bs = mb->bs;
ierr = MatGetOwnershipRange(snes->jacobian_pre, &rstart, &rend);CHKERRQ(ierr);
ierr = MatGetLocalSize(snes->jacobian_pre, PETSC_NULL, &ccsize);CHKERRQ(ierr);
nslots = (rend - rstart)/bs;
for (i = 0; i < numBlocks; ++i) {
if (mb->defaultblocks) {
ierr = ISCreateStride(((PetscObject) snes)->comm, nslots, rstart+i, numBlocks, &blocks->is);CHKERRQ(ierr);
} else if (!blocks->is) {
if (blocks->nfields > 1) {
PetscInt *ii, j, k, nfields = blocks->nfields, *fields = blocks->fields;
ierr = PetscMalloc(nfields*nslots*sizeof(PetscInt), &ii);CHKERRQ(ierr);
for (j = 0; j < nslots; ++j) {
for (k = 0; k < nfields; ++k) {
ii[nfields*j + k] = rstart + bs*j + fields[k];
}
}
ierr = ISCreateGeneral(((PetscObject) snes)->comm, nslots*nfields, ii, PETSC_OWN_POINTER, &blocks->is);CHKERRQ(ierr);
} else {
ierr = ISCreateStride(((PetscObject) snes)->comm, nslots, rstart+blocks->fields[0], bs, &blocks->is);CHKERRQ(ierr);
}
}
ierr = ISSorted(blocks->is, &sorted);CHKERRQ(ierr);
if (!sorted) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_USER, "Fields must be sorted when creating split");
blocks = blocks->next;
}
}
#if 0
/* Create matrices */
ilink = jac->head;
if (!jac->pmat) {
ierr = PetscMalloc(nsplit*sizeof(Mat),&jac->pmat);CHKERRQ(ierr);
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->pmat,ilink->is,ilink->is,MAT_INITIAL_MATRIX,&jac->pmat[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
} else {
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->pmat,ilink->is,ilink->is,MAT_REUSE_MATRIX,&jac->pmat[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
}
if (jac->realdiagonal) {
ilink = jac->head;
if (!jac->mat) {
ierr = PetscMalloc(nsplit*sizeof(Mat),&jac->mat);CHKERRQ(ierr);
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->mat,ilink->is,ilink->is,MAT_INITIAL_MATRIX,&jac->mat[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
} else {
for (i=0; i<nsplit; i++) {
if (jac->mat[i]) {ierr = MatGetSubMatrix(pc->mat,ilink->is,ilink->is,MAT_REUSE_MATRIX,&jac->mat[i]);CHKERRQ(ierr);}
ilink = ilink->next;
}
}
} else {
jac->mat = jac->pmat;
}
#endif
#if 0
if (jac->type != PC_COMPOSITE_ADDITIVE && jac->type != PC_COMPOSITE_SCHUR) {
/* extract the rows of the matrix associated with each field: used for efficient computation of residual inside algorithm */
ilink = jac->head;
if (!jac->Afield) {
ierr = PetscMalloc(nsplit*sizeof(Mat),&jac->Afield);CHKERRQ(ierr);
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->mat,ilink->is,PETSC_NULL,MAT_INITIAL_MATRIX,&jac->Afield[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
} else {
for (i=0; i<nsplit; i++) {
ierr = MatGetSubMatrix(pc->mat,ilink->is,PETSC_NULL,MAT_REUSE_MATRIX,&jac->Afield[i]);CHKERRQ(ierr);
ilink = ilink->next;
}
}
}
#endif
if (mb->type == PC_COMPOSITE_SCHUR) {
#if 0
IS ccis;
PetscInt rstart,rend;
if (nsplit != 2) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_ARG_INCOMP,"To use Schur complement preconditioner you must have exactly 2 fields");
/* When extracting off-diagonal submatrices, we take complements from this range */
ierr = MatGetOwnershipRangeColumn(pc->mat,&rstart,&rend);CHKERRQ(ierr);
/* need to handle case when one is resetting up the preconditioner */
if (jac->schur) {
ilink = jac->head;
ierr = ISComplement(ilink->is,rstart,rend,&ccis);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->mat,ilink->is,ccis,MAT_REUSE_MATRIX,&jac->B);CHKERRQ(ierr);
ierr = ISDestroy(&ccis);CHKERRQ(ierr);
ilink = ilink->next;
ierr = ISComplement(ilink->is,rstart,rend,&ccis);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->mat,ilink->is,ccis,MAT_REUSE_MATRIX,&jac->C);CHKERRQ(ierr);
ierr = ISDestroy(&ccis);CHKERRQ(ierr);
ierr = MatSchurComplementUpdate(jac->schur,jac->mat[0],jac->pmat[0],jac->B,jac->C,jac->pmat[1],pc->flag);CHKERRQ(ierr);
ierr = KSPSetOperators(jac->kspschur,jac->schur,FieldSplitSchurPre(jac),pc->flag);CHKERRQ(ierr);
} else {
KSP ksp;
char schurprefix[256];
/* extract the A01 and A10 matrices */
ilink = jac->head;
ierr = ISComplement(ilink->is,rstart,rend,&ccis);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->mat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->B);CHKERRQ(ierr);
ierr = ISDestroy(&ccis);CHKERRQ(ierr);
ilink = ilink->next;
ierr = ISComplement(ilink->is,rstart,rend,&ccis);CHKERRQ(ierr);
ierr = MatGetSubMatrix(pc->mat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->C);CHKERRQ(ierr);
ierr = ISDestroy(&ccis);CHKERRQ(ierr);
/* Use mat[0] (diagonal block of the real matrix) preconditioned by pmat[0] */
ierr = MatCreateSchurComplement(jac->mat[0],jac->pmat[0],jac->B,jac->C,jac->mat[1],&jac->schur);CHKERRQ(ierr);
/* set tabbing and options prefix of KSP inside the MatSchur */
ierr = MatSchurComplementGetKSP(jac->schur,&ksp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)ksp,(PetscObject)pc,2);CHKERRQ(ierr);
ierr = PetscSNPrintf(schurprefix,sizeof(schurprefix),"%sfieldsplit_%s_",((PetscObject)pc)->prefix?((PetscObject)pc)->prefix:"",jac->head->splitname);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(ksp,schurprefix);CHKERRQ(ierr);
ierr = MatSetFromOptions(jac->schur);CHKERRQ(ierr);
ierr = KSPCreate(((PetscObject)pc)->comm,&jac->kspschur);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)jac->kspschur);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)jac->kspschur,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = KSPSetOperators(jac->kspschur,jac->schur,FieldSplitSchurPre(jac),DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
if (jac->schurpre == PC_FIELDSPLIT_SCHUR_PRE_SELF) {
PC pc;
ierr = KSPGetPC(jac->kspschur,&pc);CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE);CHKERRQ(ierr);
/* Note: This is bad if there exist preconditioners for MATSCHURCOMPLEMENT */
}
ierr = PetscSNPrintf(schurprefix,sizeof(schurprefix),"%sfieldsplit_%s_",((PetscObject)pc)->prefix?((PetscObject)pc)->prefix:"",ilink->splitname);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(jac->kspschur,schurprefix);CHKERRQ(ierr);
/* really want setfromoptions called in PCSetFromOptions_FieldSplit(), but it is not ready yet */
ierr = KSPSetFromOptions(jac->kspschur);CHKERRQ(ierr);
ierr = PetscMalloc2(2,Vec,&jac->x,2,Vec,&jac->y);CHKERRQ(ierr);
ierr = MatGetVecs(jac->pmat[0],&jac->x[0],&jac->y[0]);CHKERRQ(ierr);
ierr = MatGetVecs(jac->pmat[1],&jac->x[1],&jac->y[1]);CHKERRQ(ierr);
ilink = jac->head;
ilink->x = jac->x[0]; ilink->y = jac->y[0];
ilink = ilink->next;
ilink->x = jac->x[1]; ilink->y = jac->y[1];
}
#endif
} else {
/* Set up the individual SNESs */
blocks = mb->blocks;
i = 0;
while (blocks) {
/*TODO: Set these correctly */
/*ierr = SNESSetFunction(blocks->snes, blocks->x, func);CHKERRQ(ierr);*/
/*ierr = SNESSetJacobian(blocks->snes, blocks->x, jac);CHKERRQ(ierr);*/
ierr = VecDuplicate(blocks->snes->vec_sol, &blocks->x);CHKERRQ(ierr);
/* really want setfromoptions called in SNESSetFromOptions_Multiblock(), but it is not ready yet */
ierr = SNESSetFromOptions(blocks->snes);CHKERRQ(ierr);
ierr = SNESSetUp(blocks->snes);CHKERRQ(ierr);
blocks = blocks->next;
i++;
}
}
/* Compute scatter contexts needed by multiplicative versions and non-default splits */
if (!mb->blocks->sctx) {
Vec xtmp;
blocks = mb->blocks;
ierr = MatGetVecs(snes->jacobian_pre, &xtmp, PETSC_NULL);CHKERRQ(ierr);
while(blocks) {
ierr = VecScatterCreate(xtmp, blocks->is, blocks->x, PETSC_NULL, &blocks->sctx);CHKERRQ(ierr);
blocks = blocks->next;
}
ierr = VecDestroy(&xtmp);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode PCBDDCSubSchursSetUp(PCBDDCSubSchurs sub_schurs, Mat S, IS is_A_I, IS is_A_B, PetscInt ncc, IS is_cc[], PetscInt xadj[], PetscInt adjncy[], PetscInt nlayers)
{
Mat A_II,A_IB,A_BI,A_BB;
ISLocalToGlobalMapping BtoNmap,ItoNmap;
PetscBT touched;
PetscInt i,n_I,n_B,n_local,*local_numbering;
PetscBool is_sorted;
PetscErrorCode ierr;
PetscFunctionBegin;
ierr = ISSorted(is_A_I,&is_sorted);CHKERRQ(ierr);
if (!is_sorted) {
SETERRQ(PetscObjectComm((PetscObject)is_A_I),PETSC_ERR_PLIB,"IS for I dofs should be shorted");
}
ierr = ISSorted(is_A_B,&is_sorted);CHKERRQ(ierr);
if (!is_sorted) {
SETERRQ(PetscObjectComm((PetscObject)is_A_B),PETSC_ERR_PLIB,"IS for B dofs should be shorted");
}
/* get sizes */
ierr = ISGetLocalSize(is_A_I,&n_I);CHKERRQ(ierr);
ierr = ISGetLocalSize(is_A_B,&n_B);CHKERRQ(ierr);
n_local = n_I+n_B;
/* maps */
ierr = ISLocalToGlobalMappingCreateIS(is_A_B,&BtoNmap);CHKERRQ(ierr);
if (nlayers >= 0 && xadj != NULL && adjncy != NULL) { /* I problems have a different size of the original ones */
ierr = ISLocalToGlobalMappingCreateIS(is_A_I,&ItoNmap);CHKERRQ(ierr);
/* allocate some auxiliary space */
ierr = PetscMalloc1(n_local,&local_numbering);CHKERRQ(ierr);
ierr = PetscBTCreate(n_local,&touched);CHKERRQ(ierr);
} else {
ItoNmap = 0;
local_numbering = 0;
touched = 0;
}
/* get Schur complement matrices */
ierr = MatSchurComplementGetSubMatrices(S,&A_II,NULL,&A_IB,&A_BI,&A_BB);CHKERRQ(ierr);
/* allocate space for schur complements */
ierr = PetscMalloc5(ncc,&sub_schurs->is_AEj_I,ncc,&sub_schurs->is_AEj_B,ncc,&sub_schurs->S_Ej,ncc,&sub_schurs->work1,ncc,&sub_schurs->work2);CHKERRQ(ierr);
sub_schurs->n_subs = ncc;
/* cycle on subsets and extract schur complements */
for (i=0;i<sub_schurs->n_subs;i++) {
Mat AE_II,AE_IE,AE_EI,AE_EE;
IS is_I,is_subset_B;
/* get IS for subsets in B numbering */
ierr = ISDuplicate(is_cc[i],&sub_schurs->is_AEj_B[i]);CHKERRQ(ierr);
ierr = ISSort(sub_schurs->is_AEj_B[i]);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyIS(BtoNmap,IS_GTOLM_DROP,sub_schurs->is_AEj_B[i],&is_subset_B);CHKERRQ(ierr);
/* BB block on subset */
ierr = MatGetSubMatrix(A_BB,is_subset_B,is_subset_B,MAT_INITIAL_MATRIX,&AE_EE);CHKERRQ(ierr);
if (ItoNmap) { /* is ItoNmap has been computed, extracts only a part of I dofs */
const PetscInt* idx_B;
PetscInt n_local_dofs,n_prev_added,j,layer,subset_size;
/* all boundary dofs must be skipped when adding layers */
ierr = PetscBTMemzero(n_local,touched);CHKERRQ(ierr);
ierr = ISGetIndices(is_A_B,&idx_B);CHKERRQ(ierr);
for (j=0;j<n_B;j++) {
ierr = PetscBTSet(touched,idx_B[j]);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(is_A_B,&idx_B);CHKERRQ(ierr);
/* add next layers of dofs */
ierr = ISGetLocalSize(is_cc[i],&subset_size);CHKERRQ(ierr);
ierr = ISGetIndices(is_cc[i],&idx_B);CHKERRQ(ierr);
ierr = PetscMemcpy(local_numbering,idx_B,subset_size*sizeof(PetscInt));CHKERRQ(ierr);
ierr = ISRestoreIndices(is_cc[i],&idx_B);CHKERRQ(ierr);
n_local_dofs = subset_size;
n_prev_added = subset_size;
for (layer=0;layer<nlayers;layer++) {
PetscInt n_added;
if (n_local_dofs == n_I+subset_size) break;
if (n_local_dofs > n_I+subset_size) {
SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Error querying layer %d. Out of bound access (%d > %d)",layer,n_local_dofs,n_I+subset_size);
}
ierr = PCBDDCAdjGetNextLayer_Private(local_numbering+n_local_dofs,n_prev_added,touched,xadj,adjncy,&n_added);CHKERRQ(ierr);
n_prev_added = n_added;
n_local_dofs += n_added;
if (!n_added) break;
}
/* IS for I dofs in original numbering and in I numbering */
ierr = ISCreateGeneral(PetscObjectComm((PetscObject)ItoNmap),n_local_dofs-subset_size,local_numbering+subset_size,PETSC_COPY_VALUES,&sub_schurs->is_AEj_I[i]);CHKERRQ(ierr);
ierr = ISSort(sub_schurs->is_AEj_I[i]);CHKERRQ(ierr);
ierr = ISGlobalToLocalMappingApplyIS(ItoNmap,IS_GTOLM_DROP,sub_schurs->is_AEj_I[i],&is_I);CHKERRQ(ierr);
/* II block */
ierr = MatGetSubMatrix(A_II,is_I,is_I,MAT_INITIAL_MATRIX,&AE_II);CHKERRQ(ierr);
} else { /* in this case we can take references of already existing IS and matrices for I dofs */
/* IS for I dofs in original numbering */
ierr = PetscObjectReference((PetscObject)is_A_I);CHKERRQ(ierr);
sub_schurs->is_AEj_I[i] = is_A_I;
/* IS for I dofs in I numbering TODO: "first" argument of ISCreateStride is not general */
ierr = ISCreateStride(PetscObjectComm((PetscObject)is_A_I),n_I,0,1,&is_I);CHKERRQ(ierr);
/* II block is the same */
ierr = PetscObjectReference((PetscObject)A_II);CHKERRQ(ierr);
AE_II = A_II;
}
/* IE block */
ierr = MatGetSubMatrix(A_IB,is_I,is_subset_B,MAT_INITIAL_MATRIX,&AE_IE);CHKERRQ(ierr);
/* EI block */
ierr = MatGetSubMatrix(A_BI,is_subset_B,is_I,MAT_INITIAL_MATRIX,&AE_EI);CHKERRQ(ierr);
/* setup Schur complements on subset */
ierr = MatCreateSchurComplement(AE_II,AE_II,AE_IE,AE_EI,AE_EE,&sub_schurs->S_Ej[i]);CHKERRQ(ierr);
ierr = MatGetVecs(sub_schurs->S_Ej[i],&sub_schurs->work1[i],&sub_schurs->work2[i]);CHKERRQ(ierr);
if (AE_II == A_II) { /* we can reuse the same ksp */
KSP ksp;
ierr = MatSchurComplementGetKSP(S,&ksp);CHKERRQ(ierr);
ierr = MatSchurComplementSetKSP(sub_schurs->S_Ej[i],ksp);CHKERRQ(ierr);
} else { /* build new ksp object which inherits ksp and pc types from the original one */
KSP origksp,schurksp;
PC origpc,schurpc;
KSPType ksp_type;
PCType pc_type;
PetscInt n_internal;
ierr = MatSchurComplementGetKSP(S,&origksp);CHKERRQ(ierr);
ierr = MatSchurComplementGetKSP(sub_schurs->S_Ej[i],&schurksp);CHKERRQ(ierr);
ierr = KSPGetType(origksp,&ksp_type);CHKERRQ(ierr);
ierr = KSPSetType(schurksp,ksp_type);CHKERRQ(ierr);
ierr = KSPGetPC(schurksp,&schurpc);CHKERRQ(ierr);
ierr = KSPGetPC(origksp,&origpc);CHKERRQ(ierr);
ierr = PCGetType(origpc,&pc_type);CHKERRQ(ierr);
ierr = PCSetType(schurpc,pc_type);CHKERRQ(ierr);
ierr = ISGetSize(is_I,&n_internal);CHKERRQ(ierr);
if (n_internal) { /* UMFPACK gives error with 0 sized problems */
MatSolverPackage solver=NULL;
ierr = PCFactorGetMatSolverPackage(origpc,(const MatSolverPackage*)&solver);CHKERRQ(ierr);
if (solver) {
ierr = PCFactorSetMatSolverPackage(schurpc,solver);CHKERRQ(ierr);
}
}
ierr = KSPSetUp(schurksp);CHKERRQ(ierr);
}
/* free */
ierr = MatDestroy(&AE_II);CHKERRQ(ierr);
ierr = MatDestroy(&AE_EE);CHKERRQ(ierr);
ierr = MatDestroy(&AE_IE);CHKERRQ(ierr);
ierr = MatDestroy(&AE_EI);CHKERRQ(ierr);
ierr = ISDestroy(&is_I);CHKERRQ(ierr);
ierr = ISDestroy(&is_subset_B);CHKERRQ(ierr);
}
/* free */
ierr = ISLocalToGlobalMappingDestroy(&ItoNmap);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingDestroy(&BtoNmap);CHKERRQ(ierr);
ierr = PetscFree(local_numbering);CHKERRQ(ierr);
ierr = PetscBTDestroy(&touched);CHKERRQ(ierr);
PetscFunctionReturn(0);
}