static PetscErrorCode PCSetUp_SVD(PC pc)
{
#if defined(PETSC_MISSING_LAPACK_GESVD)
SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"GESVD - Lapack routine is unavailable\nNot able to provide singular value estimates.");
#else
PC_SVD *jac = (PC_SVD*)pc->data;
PetscErrorCode ierr;
PetscScalar *a,*u,*v,*d,*work;
PetscBLASInt nb,lwork;
PetscInt i,n;
PetscMPIInt size;
PetscFunctionBegin;
ierr = MatDestroy(&jac->A);CHKERRQ(ierr);
ierr = MPI_Comm_size(((PetscObject)pc->pmat)->comm,&size);CHKERRQ(ierr);
if (size > 1) {
Mat redmat;
PetscInt M;
ierr = MatGetSize(pc->pmat,&M,NULL);CHKERRQ(ierr);
ierr = MatGetRedundantMatrix(pc->pmat,size,PETSC_COMM_SELF,M,MAT_INITIAL_MATRIX,&redmat);CHKERRQ(ierr);
ierr = MatConvert(redmat,MATSEQDENSE,MAT_INITIAL_MATRIX,&jac->A);CHKERRQ(ierr);
ierr = MatDestroy(&redmat);CHKERRQ(ierr);
} else {
ierr = MatConvert(pc->pmat,MATSEQDENSE,MAT_INITIAL_MATRIX,&jac->A);CHKERRQ(ierr);
}
if (!jac->diag) { /* assume square matrices */
ierr = MatGetVecs(jac->A,&jac->diag,&jac->work);CHKERRQ(ierr);
}
if (!jac->U) {
ierr = MatDuplicate(jac->A,MAT_DO_NOT_COPY_VALUES,&jac->U);CHKERRQ(ierr);
ierr = MatDuplicate(jac->A,MAT_DO_NOT_COPY_VALUES,&jac->Vt);CHKERRQ(ierr);
}
ierr = MatGetSize(pc->pmat,&n,NULL);CHKERRQ(ierr);
ierr = PetscBLASIntCast(n,&nb);CHKERRQ(ierr);
lwork = 5*nb;
ierr = PetscMalloc(lwork*sizeof(PetscScalar),&work);CHKERRQ(ierr);
ierr = MatDenseGetArray(jac->A,&a);CHKERRQ(ierr);
ierr = MatDenseGetArray(jac->U,&u);CHKERRQ(ierr);
ierr = MatDenseGetArray(jac->Vt,&v);CHKERRQ(ierr);
ierr = VecGetArray(jac->diag,&d);CHKERRQ(ierr);
#if !defined(PETSC_USE_COMPLEX)
{
PetscBLASInt lierr;
ierr = PetscFPTrapPush(PETSC_FP_TRAP_OFF);CHKERRQ(ierr);
PetscStackCall("LAPACKgesvd",LAPACKgesvd_("A","A",&nb,&nb,a,&nb,d,u,&nb,v,&nb,work,&lwork,&lierr));
if (lierr) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"gesv() error %d",lierr);
ierr = PetscFPTrapPop();CHKERRQ(ierr);
}
#else
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Not coded for complex");
#endif
ierr = MatDenseRestoreArray(jac->A,&a);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(jac->U,&u);CHKERRQ(ierr);
ierr = MatDenseRestoreArray(jac->Vt,&v);CHKERRQ(ierr);
for (i=n-1; i>=0; i--) if (PetscRealPart(d[i]) > jac->zerosing) break;
jac->nzero = n-1-i;
if (jac->monitor) {
ierr = PetscViewerASCIIAddTab(jac->monitor,((PetscObject)pc)->tablevel);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(jac->monitor," SVD: condition number %14.12e, %D of %D singular values are (nearly) zero\n",(double)PetscRealPart(d[0]/d[n-1]),jac->nzero,n);CHKERRQ(ierr);
if (n >= 10) { /* print 5 smallest and 5 largest */
ierr = PetscViewerASCIIPrintf(jac->monitor," SVD: smallest singular values: %14.12e %14.12e %14.12e %14.12e %14.12e\n",(double)PetscRealPart(d[n-1]),(double)PetscRealPart(d[n-2]),(double)PetscRealPart(d[n-3]),(double)PetscRealPart(d[n-4]),(double)PetscRealPart(d[n-5]));CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(jac->monitor," SVD: largest singular values : %14.12e %14.12e %14.12e %14.12e %14.12e\n",(double)PetscRealPart(d[4]),(double)PetscRealPart(d[3]),(double)PetscRealPart(d[2]),(double)PetscRealPart(d[1]),(double)PetscRealPart(d[0]));CHKERRQ(ierr);
} else { /* print all singular values */
char buf[256],*p;
size_t left = sizeof(buf),used;
PetscInt thisline;
for (p=buf,i=n-1,thisline=1; i>=0; i--,thisline++) {
ierr = PetscSNPrintfCount(p,left," %14.12e",&used,(double)PetscRealPart(d[i]));CHKERRQ(ierr);
left -= used;
p += used;
if (thisline > 4 || i==0) {
ierr = PetscViewerASCIIPrintf(jac->monitor," SVD: singular values:%s\n",buf);CHKERRQ(ierr);
p = buf;
thisline = 0;
}
}
}
ierr = PetscViewerASCIISubtractTab(jac->monitor,((PetscObject)pc)->tablevel);CHKERRQ(ierr);
}
ierr = PetscInfo2(pc,"Largest and smallest singular values %14.12e %14.12e\n",(double)PetscRealPart(d[0]),(double)PetscRealPart(d[n-1]));CHKERRQ(ierr);
for (i=0; i<n-jac->nzero; i++) d[i] = 1.0/d[i];
for (; i<n; i++) d[i] = 0.0;
if (jac->essrank > 0) for (i=0; i<n-jac->nzero-jac->essrank; i++) d[i] = 0.0; /* Skip all but essrank eigenvalues */
ierr = PetscInfo1(pc,"Number of zero or nearly singular values %D\n",jac->nzero);CHKERRQ(ierr);
ierr = VecRestoreArray(jac->diag,&d);CHKERRQ(ierr);
#if defined(foo)
{
PetscViewer viewer;
ierr = PetscViewerBinaryOpen(PETSC_COMM_SELF,"joe",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
ierr = MatView(jac->A,viewer);CHKERRQ(ierr);
ierr = MatView(jac->U,viewer);CHKERRQ(ierr);
ierr = MatView(jac->Vt,viewer);CHKERRQ(ierr);
ierr = VecView(jac->diag,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(viewer);CHKERRQ(ierr);
}
#endif
ierr = PetscFree(work);CHKERRQ(ierr);
PetscFunctionReturn(0);
#endif
}
static PetscErrorCode PCSetUp_Redundant(PC pc)
{
PC_Redundant *red = (PC_Redundant*)pc->data;
PetscErrorCode ierr;
PetscInt mstart,mend,mlocal,M;
PetscMPIInt size;
MPI_Comm comm,subcomm;
Vec x;
const char *prefix;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr);
/* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size == 1) red->useparallelmat = PETSC_FALSE;
if (!pc->setupcalled) {
PetscInt mloc_sub;
if (!red->psubcomm) {
ierr = PetscSubcommCreate(comm,&red->psubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetNumber(red->psubcomm,red->nsubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetType(red->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr);
/* enable runtime switch of psubcomm type, e.g., '-psubcomm_type interlaced */
ierr = PetscSubcommSetFromOptions(red->psubcomm);CHKERRQ(ierr);
ierr = PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));CHKERRQ(ierr);
/* create a new PC that processors in each subcomm have copy of */
subcomm = red->psubcomm->comm;
ierr = KSPCreate(subcomm,&red->ksp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)red->ksp);CHKERRQ(ierr);
ierr = KSPSetType(red->ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(red->ksp,&red->pc);CHKERRQ(ierr);
ierr = PCSetType(red->pc,PCLU);CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(red->ksp,prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(red->ksp,"redundant_");CHKERRQ(ierr);
} else {
subcomm = red->psubcomm->comm;
}
if (red->useparallelmat) {
/* grab the parallel matrix and put it into processors of a subcomminicator */
ierr = MatGetRedundantMatrix(pc->pmat,red->psubcomm->n,subcomm,MAT_INITIAL_MATRIX,&red->pmats);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr);
/* get working vectors xsub and ysub */
ierr = MatGetVecs(red->pmats,&red->xsub,&red->ysub);CHKERRQ(ierr);
/* create working vectors xdup and ydup.
xdup concatenates all xsub's contigously to form a mpi vector over dupcomm (see PetscSubcommCreate_interlaced())
ydup concatenates all ysub and has empty local arrays because ysub's arrays will be place into it.
Note: we use communicator dupcomm, not PetscObjectComm((PetscObject)pc)! */
ierr = MatGetLocalSize(red->pmats,&mloc_sub,NULL);CHKERRQ(ierr);
ierr = VecCreateMPI(red->psubcomm->dupparent,mloc_sub,PETSC_DECIDE,&red->xdup);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(red->psubcomm->dupparent,1,mloc_sub,PETSC_DECIDE,NULL,&red->ydup);CHKERRQ(ierr);
/* create vecscatters */
if (!red->scatterin) { /* efficiency of scatterin is independent from psubcomm_type! */
IS is1,is2;
PetscInt *idx1,*idx2,i,j,k;
ierr = MatGetVecs(pc->pmat,&x,0);CHKERRQ(ierr);
ierr = VecGetSize(x,&M);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(x,&mstart,&mend);CHKERRQ(ierr);
mlocal = mend - mstart;
ierr = PetscMalloc2(red->psubcomm->n*mlocal,&idx1,red->psubcomm->n*mlocal,&idx2);CHKERRQ(ierr);
j = 0;
for (k=0; k<red->psubcomm->n; k++) {
for (i=mstart; i<mend; i++) {
idx1[j] = i;
idx2[j++] = i + M*k;
}
}
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(x,is1,red->xdup,is2,&red->scatterin);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
/* Impl below is good for PETSC_SUBCOMM_INTERLACED (no inter-process communication) and PETSC_SUBCOMM_CONTIGUOUS (communication within subcomm) */
ierr = ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*M,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(comm,mlocal,mstart,1,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(red->xdup,is1,x,is2,&red->scatterout);CHKERRQ(ierr);
ierr = ISDestroy(&is1);CHKERRQ(ierr);
ierr = ISDestroy(&is2);CHKERRQ(ierr);
ierr = PetscFree2(idx1,idx2);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
}
} else { /* !red->useparallelmat */
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr);
}
} else { /* pc->setupcalled */
if (red->useparallelmat) {
MatReuse reuse;
/* grab the parallel matrix and put it into processors of a subcomminicator */
/*--------------------------------------------------------------------------*/
if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
/* destroy old matrices */
ierr = MatDestroy(&red->pmats);CHKERRQ(ierr);
reuse = MAT_INITIAL_MATRIX;
} else {
reuse = MAT_REUSE_MATRIX;
}
ierr = MatGetRedundantMatrix(pc->pmat,red->psubcomm->n,red->psubcomm->comm,reuse,&red->pmats);CHKERRQ(ierr);
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr);
} else { /* !red->useparallelmat */
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr);
}
}
if (pc->setfromoptionscalled) {
ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr);
}
ierr = KSPSetUp(red->ksp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
KSP subksp;
Mat A,subA;
Vec x,b,u,subb,subx,subu;
PetscViewer fd;
char file[PETSC_MAX_PATH_LEN];
PetscBool flg;
PetscErrorCode ierr;
PetscInt i,m,n,its;
PetscReal norm;
PetscMPIInt rank,size;
MPI_Comm comm,subcomm;
PetscSubcomm psubcomm;
PetscInt nsubcomm=1,id;
PetscScalar *barray,*xarray,*uarray,*array,one=1.0;
PetscInt type=1;
PetscInitialize(&argc,&args,(char*)0,help);
/* Load the matrix */
ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,1,"Must indicate binary file with the -f option");
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
/* Load the matrix; then destroy the viewer.*/
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)A,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
/* Create rhs vector b */
ierr = MatGetLocalSize(A,&m,NULL);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
ierr = VecSet(b,one);CHKERRQ(ierr);
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&u);CHKERRQ(ierr);
ierr = VecSet(x,0.0);CHKERRQ(ierr);
/* Test MatGetMultiProcBlock() */
ierr = PetscOptionsGetInt(NULL,"-nsubcomm",&nsubcomm,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-subcomm_type",&type,NULL);CHKERRQ(ierr);
ierr = PetscSubcommCreate(comm,&psubcomm);CHKERRQ(ierr);
ierr = PetscSubcommSetNumber(psubcomm,nsubcomm);CHKERRQ(ierr);
if (type == PETSC_SUBCOMM_GENERAL) { /* user provides color, subrank and duprank */
PetscMPIInt color,subrank,duprank,subsize;
duprank = size-1 - rank;
subsize = size/nsubcomm;
if (subsize*nsubcomm != size) SETERRQ2(comm,PETSC_ERR_SUP,"This example requires nsubcomm %D divides size %D",nsubcomm,size);
color = duprank/subsize;
subrank = duprank - color*subsize;
ierr = PetscSubcommSetTypeGeneral(psubcomm,color,subrank);CHKERRQ(ierr);
} else if (type == PETSC_SUBCOMM_CONTIGUOUS) {
ierr = PetscSubcommSetType(psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr);
} else if (type == PETSC_SUBCOMM_INTERLACED) {
ierr = PetscSubcommSetType(psubcomm,PETSC_SUBCOMM_INTERLACED);CHKERRQ(ierr);
} else SETERRQ1(psubcomm->parent,PETSC_ERR_SUP,"PetscSubcommType %D is not supported yet",type);
ierr = PetscSubcommSetFromOptions(psubcomm);CHKERRQ(ierr);
subcomm = psubcomm->comm;
/* Test MatGetRedundantMatrix() */
if (size > 1) {
ierr = MatGetRedundantMatrix(A,nsubcomm,subcomm,MAT_INITIAL_MATRIX,&subA);CHKERRQ(ierr);
ierr = MatGetRedundantMatrix(A,nsubcomm,subcomm,MAT_REUSE_MATRIX,&subA);CHKERRQ(ierr);
ierr = MatDestroy(&subA);CHKERRQ(ierr);
}
/* Create subA */
ierr = MatGetMultiProcBlock(A,subcomm,MAT_INITIAL_MATRIX,&subA);CHKERRQ(ierr);
ierr = MatGetMultiProcBlock(A,subcomm,MAT_REUSE_MATRIX,&subA);CHKERRQ(ierr);
/* Create sub vectors without arrays. Place b's and x's local arrays into subb and subx */
ierr = MatGetLocalSize(subA,&m,&n);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(subcomm,1,m,PETSC_DECIDE,NULL,&subb);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(subcomm,1,n,PETSC_DECIDE,NULL,&subx);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(subcomm,1,n,PETSC_DECIDE,NULL,&subu);CHKERRQ(ierr);
ierr = VecGetArray(b,&barray);CHKERRQ(ierr);
ierr = VecGetArray(x,&xarray);CHKERRQ(ierr);
ierr = VecGetArray(u,&uarray);CHKERRQ(ierr);
ierr = VecPlaceArray(subb,barray);CHKERRQ(ierr);
ierr = VecPlaceArray(subx,xarray);CHKERRQ(ierr);
ierr = VecPlaceArray(subu,uarray);CHKERRQ(ierr);
/* Create linear solvers associated with subA */
ierr = KSPCreate(subcomm,&subksp);CHKERRQ(ierr);
ierr = KSPSetOperators(subksp,subA,subA);CHKERRQ(ierr);
ierr = KSPSetFromOptions(subksp);CHKERRQ(ierr);
/* Solve sub systems */
ierr = KSPSolve(subksp,subb,subx);CHKERRQ(ierr);
ierr = KSPGetIterationNumber(subksp,&its);CHKERRQ(ierr);
/* check residual */
ierr = MatMult(subA,subx,subu);CHKERRQ(ierr);
ierr = VecAXPY(subu,-1.0,subb);CHKERRQ(ierr);
ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr);
if (norm > 1.e-4 && !rank) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"[%D] Number of iterations = %3D\n",rank,its);CHKERRQ(ierr);
printf("Error: Residual norm of each block |subb - subA*subx |= %g\n",(double)norm);
}
ierr = VecResetArray(subb);CHKERRQ(ierr);
ierr = VecResetArray(subx);CHKERRQ(ierr);
ierr = VecResetArray(subu);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-subvec_view",&id,&flg);CHKERRQ(ierr);
if (flg && rank == id) {
ierr = PetscPrintf(PETSC_COMM_SELF,"[%D] subb:\n", rank);
ierr = VecGetArray(subb,&array);CHKERRQ(ierr);
for (i=0; i<m; i++) printf("%g\n",(double)PetscRealPart(array[i]));
ierr = VecRestoreArray(subb,&array);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"[%D] subx:\n", rank);
ierr = VecGetArray(subx,&array);CHKERRQ(ierr);
for (i=0; i<m; i++) printf("%g\n",(double)PetscRealPart(array[i]));
ierr = VecRestoreArray(subx,&array);CHKERRQ(ierr);
}
ierr = VecRestoreArray(x,&xarray);CHKERRQ(ierr);
ierr = VecRestoreArray(b,&barray);CHKERRQ(ierr);
ierr = VecRestoreArray(u,&uarray);CHKERRQ(ierr);
ierr = MatDestroy(&subA);CHKERRQ(ierr);
ierr = VecDestroy(&subb);CHKERRQ(ierr);
ierr = VecDestroy(&subx);CHKERRQ(ierr);
ierr = VecDestroy(&subu);CHKERRQ(ierr);
ierr = KSPDestroy(&subksp);CHKERRQ(ierr);
ierr = PetscSubcommDestroy(&psubcomm);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
static PetscErrorCode PCSetUp_Redundant(PC pc)
{
PC_Redundant *red = (PC_Redundant*)pc->data;
PetscErrorCode ierr;
PetscInt mstart,mend,mlocal,m,mlocal_sub,rstart_sub,rend_sub,mloc_sub;
PetscMPIInt size;
MatReuse reuse = MAT_INITIAL_MATRIX;
MatStructure str = DIFFERENT_NONZERO_PATTERN;
MPI_Comm comm = ((PetscObject)pc)->comm,subcomm;
Vec vec;
PetscMPIInt subsize,subrank;
const char *prefix;
const PetscInt *range;
PetscFunctionBegin;
ierr = MatGetVecs(pc->pmat,&vec,0);CHKERRQ(ierr);
ierr = VecGetSize(vec,&m);CHKERRQ(ierr);
if (!pc->setupcalled) {
if (!red->psubcomm) {
ierr = PetscSubcommCreate(comm,red->nsubcomm,&red->psubcomm);CHKERRQ(ierr);
ierr = PetscLogObjectMemory(pc,sizeof(PetscSubcomm));CHKERRQ(ierr);
/* create a new PC that processors in each subcomm have copy of */
subcomm = red->psubcomm->comm;
ierr = KSPCreate(subcomm,&red->ksp);CHKERRQ(ierr);
ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr);
ierr = PetscLogObjectParent(pc,red->ksp);CHKERRQ(ierr);
ierr = KSPSetType(red->ksp,KSPPREONLY);CHKERRQ(ierr);
ierr = KSPGetPC(red->ksp,&red->pc);CHKERRQ(ierr);
ierr = PCSetType(red->pc,PCLU);CHKERRQ(ierr);
ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr);
ierr = KSPSetOptionsPrefix(red->ksp,prefix);CHKERRQ(ierr);
ierr = KSPAppendOptionsPrefix(red->ksp,"redundant_");CHKERRQ(ierr);
} else {
subcomm = red->psubcomm->comm;
}
/* create working vectors xsub/ysub and xdup/ydup */
ierr = VecGetLocalSize(vec,&mlocal);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(vec,&mstart,&mend);CHKERRQ(ierr);
/* get local size of xsub/ysub */
ierr = MPI_Comm_size(subcomm,&subsize);CHKERRQ(ierr);
ierr = MPI_Comm_rank(subcomm,&subrank);CHKERRQ(ierr);
ierr = MatGetOwnershipRanges(pc->pmat,&range);CHKERRQ(ierr);
rstart_sub = range[red->psubcomm->n*subrank]; /* rstart in xsub/ysub */
if (subrank+1 < subsize){
rend_sub = range[red->psubcomm->n*(subrank+1)];
} else {
rend_sub = m;
}
mloc_sub = rend_sub - rstart_sub;
ierr = VecCreateMPI(subcomm,mloc_sub,PETSC_DECIDE,&red->ysub);CHKERRQ(ierr);
/* create xsub with empty local arrays, because xdup's arrays will be placed into it */
ierr = VecCreateMPIWithArray(subcomm,mloc_sub,PETSC_DECIDE,PETSC_NULL,&red->xsub);CHKERRQ(ierr);
/* create xdup and ydup. ydup has empty local arrays because ysub's arrays will be place into it.
Note: we use communicator dupcomm, not ((PetscObject)pc)->comm! */
ierr = VecCreateMPI(red->psubcomm->dupparent,mloc_sub,PETSC_DECIDE,&red->xdup);CHKERRQ(ierr);
ierr = VecCreateMPIWithArray(red->psubcomm->dupparent,mloc_sub,PETSC_DECIDE,PETSC_NULL,&red->ydup);CHKERRQ(ierr);
/* create vec scatters */
if (!red->scatterin){
IS is1,is2;
PetscInt *idx1,*idx2,i,j,k;
ierr = PetscMalloc2(red->psubcomm->n*mlocal,PetscInt,&idx1,red->psubcomm->n*mlocal,PetscInt,&idx2);CHKERRQ(ierr);
j = 0;
for (k=0; k<red->psubcomm->n; k++){
for (i=mstart; i<mend; i++){
idx1[j] = i;
idx2[j++] = i + m*k;
}
}
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,&is1);CHKERRQ(ierr);
ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(vec,is1,red->xdup,is2,&red->scatterin);CHKERRQ(ierr);
ierr = ISDestroy(is1);CHKERRQ(ierr);
ierr = ISDestroy(is2);CHKERRQ(ierr);
ierr = ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*m,1,&is1);CHKERRQ(ierr);
ierr = ISCreateStride(comm,mlocal,mstart,1,&is2);CHKERRQ(ierr);
ierr = VecScatterCreate(red->xdup,is1,vec,is2,&red->scatterout);CHKERRQ(ierr);
ierr = ISDestroy(is1);CHKERRQ(ierr);
ierr = ISDestroy(is2);CHKERRQ(ierr);
ierr = PetscFree2(idx1,idx2);CHKERRQ(ierr);
}
}
ierr = VecDestroy(vec);CHKERRQ(ierr);
/* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
if (size == 1) {
red->useparallelmat = PETSC_FALSE;
}
if (red->useparallelmat) {
if (pc->setupcalled == 1 && pc->flag == DIFFERENT_NONZERO_PATTERN) {
/* destroy old matrices */
if (red->pmats) {
ierr = MatDestroy(red->pmats);CHKERRQ(ierr);
}
} else if (pc->setupcalled == 1) {
reuse = MAT_REUSE_MATRIX;
str = SAME_NONZERO_PATTERN;
}
/* grab the parallel matrix and put it into processors of a subcomminicator */
/*--------------------------------------------------------------------------*/
ierr = VecGetLocalSize(red->ysub,&mlocal_sub);CHKERRQ(ierr);
ierr = MatGetRedundantMatrix(pc->pmat,red->psubcomm->n,red->psubcomm->comm,mlocal_sub,reuse,&red->pmats);CHKERRQ(ierr);
/* tell PC of the subcommunicator its operators */
ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats,str);CHKERRQ(ierr);
} else {
ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat,pc->flag);CHKERRQ(ierr);
}
if (pc->setfromoptionscalled){
ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr);
}
ierr = KSPSetUp(red->ksp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
