int main(int argc,char **argv) { PetscErrorCode ierr; PetscViewer viewer,subviewer,subsubviewer; PetscViewerFormat format; PetscBool flg; PetscSubcomm psubcomm,psubsubcomm; MPI_Comm comm,subcomm,subsubcomm; PetscMPIInt size; /* Every PETSc routine should begin with the PetscInitialize() routine. argc, argv - These command line arguments are taken to extract the options supplied to PETSc and options supplied to MPI. help - When PETSc executable is invoked with the option -help, it prints the various options that can be applied at runtime. The user can use the "help" variable place additional help messages in this printout. */ ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; comm = PETSC_COMM_WORLD; ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); if (size < 4) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Must run with at least 4 MPI processes"); ierr = PetscOptionsGetViewer(comm,NULL,"-viewer",&viewer,&format,&flg);CHKERRQ(ierr); if (!viewer) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"Must use -viewer option"); ierr = PetscViewerASCIIPrintf(viewer,"Print called on original full viewer %d\n",PetscGlobalRank);CHKERRQ(ierr); ierr = PetscSubcommCreate(comm,&psubcomm);CHKERRQ(ierr); ierr = PetscSubcommSetNumber(psubcomm,2);CHKERRQ(ierr); ierr = PetscSubcommSetType(psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr); /* enable runtime switch of psubcomm type, e.g., '-psubcomm_type interlaced */ ierr = PetscSubcommSetFromOptions(psubcomm);CHKERRQ(ierr); subcomm = PetscSubcommChild(psubcomm); ierr = PetscViewerGetSubViewer(viewer,subcomm,&subviewer);CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(subviewer," Print called on sub viewers %d\n",PetscGlobalRank);CHKERRQ(ierr); ierr = PetscSubcommCreate(subcomm,&psubsubcomm);CHKERRQ(ierr); ierr = PetscSubcommSetNumber(psubsubcomm,2);CHKERRQ(ierr); ierr = PetscSubcommSetType(psubsubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr); /* enable runtime switch of psubcomm type, e.g., '-psubcomm_type interlaced */ ierr = PetscSubcommSetFromOptions(psubsubcomm);CHKERRQ(ierr); subsubcomm = PetscSubcommChild(psubsubcomm); ierr = PetscViewerGetSubViewer(subviewer,subsubcomm,&subsubviewer);CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(subsubviewer," Print called on sub sub viewers %d\n",PetscGlobalRank);CHKERRQ(ierr); ierr = PetscViewerRestoreSubViewer(subviewer,subsubcomm,&subsubviewer);CHKERRQ(ierr); ierr = PetscViewerRestoreSubViewer(viewer,subcomm,&subviewer);CHKERRQ(ierr); ierr = PetscSubcommDestroy(&psubsubcomm);CHKERRQ(ierr); ierr = PetscSubcommDestroy(&psubcomm);CHKERRQ(ierr); ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
int main(int argc,char **args) { Mat A,Ar,C; PetscViewer fd; /* viewer */ char file[PETSC_MAX_PATH_LEN]; /* input file name */ PetscErrorCode ierr; PetscInt ns=2,np; PetscSubcomm subc; PetscBool flg; PetscInitialize(&argc,&args,(char*)0,help); /* Determine files from which we read the two linear systems (matrix and right-hand-side vector). */ ierr = PetscOptionsGetString(NULL,"-f0",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_WORLD,1,"Must indicate binary file with the -f0 option"); ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&np);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Reading matrix with %d processors\n",np);CHKERRQ(ierr); ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatLoad(A,fd);CHKERRQ(ierr); ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr); /* Determines amount of subcomunicators */ ierr = PetscOptionsGetInt(NULL,"-nsub",&ns,NULL);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Splitting in %d subcommunicators\n",ns);CHKERRQ(ierr); ierr = PetscSubcommCreate(PetscObjectComm((PetscObject)A),&subc);CHKERRQ(ierr); ierr = PetscSubcommSetNumber(subc,ns);CHKERRQ(ierr); ierr = PetscSubcommSetType(subc,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr); ierr = PetscSubcommSetFromOptions(subc);CHKERRQ(ierr); ierr = MatCreateRedundantMatrix(A,0,PetscSubcommChild(subc),MAT_INITIAL_MATRIX,&Ar);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_WORLD,"Copying matrix\n",ns);CHKERRQ(ierr); ierr = MatDuplicate(Ar,MAT_COPY_VALUES,&C);CHKERRQ(ierr); ierr = PetscSubcommDestroy(&subc);CHKERRQ(ierr); /* Free memory */ ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = MatDestroy(&Ar);CHKERRQ(ierr); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }
static PetscErrorCode PCRedundantGetKSP_Redundant(PC pc,KSP *innerksp) { PetscErrorCode ierr; PC_Redundant *red = (PC_Redundant*)pc->data; MPI_Comm comm,subcomm; const char *prefix; PetscFunctionBegin; if (!red->psubcomm) { ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr); ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr); ierr = PetscSubcommCreate(comm,&red->psubcomm);CHKERRQ(ierr); ierr = PetscSubcommSetNumber(red->psubcomm,red->nsubcomm);CHKERRQ(ierr); ierr = PetscSubcommSetType(red->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr); ierr = PetscSubcommSetOptionsPrefix(red->psubcomm,prefix);CHKERRQ(ierr); ierr = PetscSubcommSetFromOptions(red->psubcomm);CHKERRQ(ierr); ierr = PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));CHKERRQ(ierr); /* create a new PC that processors in each subcomm have copy of */ subcomm = PetscSubcommChild(red->psubcomm); ierr = KSPCreate(subcomm,&red->ksp);CHKERRQ(ierr); ierr = KSPSetErrorIfNotConverged(red->ksp,pc->erroriffailure);CHKERRQ(ierr); ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr); ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)red->ksp);CHKERRQ(ierr); ierr = KSPSetType(red->ksp,KSPPREONLY);CHKERRQ(ierr); ierr = KSPGetPC(red->ksp,&red->pc);CHKERRQ(ierr); ierr = PCSetType(red->pc,PCLU);CHKERRQ(ierr); if (red->shifttypeset) { ierr = PCFactorSetShiftType(red->pc,red->shifttype);CHKERRQ(ierr); red->shifttypeset = PETSC_FALSE; } ierr = KSPSetOptionsPrefix(red->ksp,prefix);CHKERRQ(ierr); ierr = KSPAppendOptionsPrefix(red->ksp,"redundant_");CHKERRQ(ierr); } *innerksp = red->ksp; PetscFunctionReturn(0); }
static PetscErrorCode PCSetUp_Redundant(PC pc) { PC_Redundant *red = (PC_Redundant*)pc->data; PetscErrorCode ierr; PetscInt mstart,mend,mlocal,M; PetscMPIInt size; MPI_Comm comm,subcomm; Vec x; const char *prefix; PetscFunctionBegin; ierr = PetscObjectGetComm((PetscObject)pc,&comm);CHKERRQ(ierr); /* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */ ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); if (size == 1) red->useparallelmat = PETSC_FALSE; if (!pc->setupcalled) { PetscInt mloc_sub; if (!red->psubcomm) { ierr = PetscSubcommCreate(comm,&red->psubcomm);CHKERRQ(ierr); ierr = PetscSubcommSetNumber(red->psubcomm,red->nsubcomm);CHKERRQ(ierr); ierr = PetscSubcommSetType(red->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr); /* enable runtime switch of psubcomm type, e.g., '-psubcomm_type interlaced */ ierr = PetscSubcommSetFromOptions(red->psubcomm);CHKERRQ(ierr); ierr = PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));CHKERRQ(ierr); /* create a new PC that processors in each subcomm have copy of */ subcomm = PetscSubcommChild(red->psubcomm); ierr = KSPCreate(subcomm,&red->ksp);CHKERRQ(ierr); ierr = KSPSetErrorIfNotConverged(red->ksp,pc->erroriffailure);CHKERRQ(ierr); ierr = PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);CHKERRQ(ierr); ierr = PetscLogObjectParent((PetscObject)pc,(PetscObject)red->ksp);CHKERRQ(ierr); ierr = KSPSetType(red->ksp,KSPPREONLY);CHKERRQ(ierr); ierr = KSPGetPC(red->ksp,&red->pc);CHKERRQ(ierr); ierr = PCSetType(red->pc,PCLU);CHKERRQ(ierr); ierr = PCGetOptionsPrefix(pc,&prefix);CHKERRQ(ierr); ierr = KSPSetOptionsPrefix(red->ksp,prefix);CHKERRQ(ierr); ierr = KSPAppendOptionsPrefix(red->ksp,"redundant_");CHKERRQ(ierr); } else { subcomm = PetscSubcommChild(red->psubcomm); } if (red->useparallelmat) { /* grab the parallel matrix and put it into processors of a subcomminicator */ ierr = MatCreateRedundantMatrix(pc->pmat,red->psubcomm->n,subcomm,MAT_INITIAL_MATRIX,&red->pmats);CHKERRQ(ierr); ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr); /* get working vectors xsub and ysub */ ierr = MatCreateVecs(red->pmats,&red->xsub,&red->ysub);CHKERRQ(ierr); /* create working vectors xdup and ydup. xdup concatenates all xsub's contigously to form a mpi vector over dupcomm (see PetscSubcommCreate_interlaced()) ydup concatenates all ysub and has empty local arrays because ysub's arrays will be place into it. Note: we use communicator dupcomm, not PetscObjectComm((PetscObject)pc)! */ ierr = MatGetLocalSize(red->pmats,&mloc_sub,NULL);CHKERRQ(ierr); ierr = VecCreateMPI(PetscSubcommContiguousParent(red->psubcomm),mloc_sub,PETSC_DECIDE,&red->xdup);CHKERRQ(ierr); ierr = VecCreateMPIWithArray(PetscSubcommContiguousParent(red->psubcomm),1,mloc_sub,PETSC_DECIDE,NULL,&red->ydup);CHKERRQ(ierr); /* create vecscatters */ if (!red->scatterin) { /* efficiency of scatterin is independent from psubcomm_type! */ IS is1,is2; PetscInt *idx1,*idx2,i,j,k; ierr = MatCreateVecs(pc->pmat,&x,0);CHKERRQ(ierr); ierr = VecGetSize(x,&M);CHKERRQ(ierr); ierr = VecGetOwnershipRange(x,&mstart,&mend);CHKERRQ(ierr); mlocal = mend - mstart; ierr = PetscMalloc2(red->psubcomm->n*mlocal,&idx1,red->psubcomm->n*mlocal,&idx2);CHKERRQ(ierr); j = 0; for (k=0; k<red->psubcomm->n; k++) { for (i=mstart; i<mend; i++) { idx1[j] = i; idx2[j++] = i + M*k; } } ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,PETSC_COPY_VALUES,&is1);CHKERRQ(ierr); ierr = ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,PETSC_COPY_VALUES,&is2);CHKERRQ(ierr); ierr = VecScatterCreate(x,is1,red->xdup,is2,&red->scatterin);CHKERRQ(ierr); ierr = ISDestroy(&is1);CHKERRQ(ierr); ierr = ISDestroy(&is2);CHKERRQ(ierr); /* Impl below is good for PETSC_SUBCOMM_INTERLACED (no inter-process communication) and PETSC_SUBCOMM_CONTIGUOUS (communication within subcomm) */ ierr = ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*M,1,&is1);CHKERRQ(ierr); ierr = ISCreateStride(comm,mlocal,mstart,1,&is2);CHKERRQ(ierr); ierr = VecScatterCreate(red->xdup,is1,x,is2,&red->scatterout);CHKERRQ(ierr); ierr = ISDestroy(&is1);CHKERRQ(ierr); ierr = ISDestroy(&is2);CHKERRQ(ierr); ierr = PetscFree2(idx1,idx2);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); } } else { /* !red->useparallelmat */ ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr); } } else { /* pc->setupcalled */ if (red->useparallelmat) { MatReuse reuse; /* grab the parallel matrix and put it into processors of a subcomminicator */ /*--------------------------------------------------------------------------*/ if (pc->flag == DIFFERENT_NONZERO_PATTERN) { /* destroy old matrices */ ierr = MatDestroy(&red->pmats);CHKERRQ(ierr); reuse = MAT_INITIAL_MATRIX; } else { reuse = MAT_REUSE_MATRIX; } ierr = MatCreateRedundantMatrix(pc->pmat,red->psubcomm->n,PetscSubcommChild(red->psubcomm),reuse,&red->pmats);CHKERRQ(ierr); ierr = KSPSetOperators(red->ksp,red->pmats,red->pmats);CHKERRQ(ierr); } else { /* !red->useparallelmat */ ierr = KSPSetOperators(red->ksp,pc->mat,pc->pmat);CHKERRQ(ierr); } } if (pc->setfromoptionscalled) { ierr = KSPSetFromOptions(red->ksp);CHKERRQ(ierr); } ierr = KSPSetUp(red->ksp);CHKERRQ(ierr); PetscFunctionReturn(0); }
int main(int argc,char **args) { Mat C,Credundant; MatInfo info; PetscMPIInt rank,size,subsize; PetscInt i,j,m = 3,n = 2,low,high,iglobal; PetscInt Ii,J,ldim,nsubcomms; PetscErrorCode ierr; PetscBool flg_info,flg_mat; PetscScalar v,one = 1.0; Vec x,y; MPI_Comm subcomm; ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr; ierr = PetscOptionsGetInt(NULL,NULL,"-m",&m,NULL);CHKERRQ(ierr); ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr); ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr); n = 2*size; ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr); ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr); ierr = MatSetFromOptions(C);CHKERRQ(ierr); ierr = MatSetUp(C);CHKERRQ(ierr); /* Create the matrix for the five point stencil, YET AGAIN */ for (i=0; i<m; i++) { for (j=2*rank; j<2*rank+2; j++) { v = -1.0; Ii = j + n*i; if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr);} v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,INSERT_VALUES);CHKERRQ(ierr); } } /* Add extra elements (to illustrate variants of MatGetInfo) */ Ii = n; J = n-2; v = 100.0; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); Ii = n-2; J = n; v = 100.0; ierr = MatSetValues(C,1,&Ii,1,&J,&v,INSERT_VALUES);CHKERRQ(ierr); ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); /* Form vectors */ ierr = MatCreateVecs(C,&x,&y);CHKERRQ(ierr); ierr = VecGetLocalSize(x,&ldim);CHKERRQ(ierr); ierr = VecGetOwnershipRange(x,&low,&high);CHKERRQ(ierr); for (i=0; i<ldim; i++) { iglobal = i + low; v = one*((PetscReal)i) + 100.0*rank; ierr = VecSetValues(x,1,&iglobal,&v,INSERT_VALUES);CHKERRQ(ierr); } ierr = VecAssemblyBegin(x);CHKERRQ(ierr); ierr = VecAssemblyEnd(x);CHKERRQ(ierr); ierr = MatMult(C,x,y);CHKERRQ(ierr); ierr = PetscOptionsHasName(NULL,NULL,"-view_info",&flg_info);CHKERRQ(ierr); if (flg_info) { ierr = PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr); ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); ierr = MatGetInfo(C,MAT_GLOBAL_SUM,&info);CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD,"matrix information (global sums):\nnonzeros = %D, allocated nonzeros = %D\n",(PetscInt)info.nz_used,(PetscInt)info.nz_allocated);CHKERRQ(ierr); ierr = MatGetInfo (C,MAT_GLOBAL_MAX,&info);CHKERRQ(ierr); ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD,"matrix information (global max):\nnonzeros = %D, allocated nonzeros = %D\n",(PetscInt)info.nz_used,(PetscInt)info.nz_allocated);CHKERRQ(ierr); } ierr = PetscOptionsHasName(NULL,NULL,"-view_mat",&flg_mat);CHKERRQ(ierr); if (flg_mat) { ierr = MatView(C,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr); } /* Test MatCreateRedundantMatrix() */ nsubcomms = size; ierr = PetscOptionsGetInt(NULL,NULL,"-nsubcomms",&nsubcomms,NULL);CHKERRQ(ierr); ierr = MatCreateRedundantMatrix(C,nsubcomms,MPI_COMM_NULL,MAT_INITIAL_MATRIX,&Credundant);CHKERRQ(ierr); ierr = MatCreateRedundantMatrix(C,nsubcomms,MPI_COMM_NULL,MAT_REUSE_MATRIX,&Credundant);CHKERRQ(ierr); ierr = PetscObjectGetComm((PetscObject)Credundant,&subcomm);CHKERRQ(ierr); ierr = MPI_Comm_size(subcomm,&subsize);CHKERRQ(ierr); if (subsize==2 && flg_mat) { ierr = PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_(subcomm),"\n[%d] Credundant:\n",rank);CHKERRQ(ierr); ierr = MatView(Credundant,PETSC_VIEWER_STDOUT_(subcomm));CHKERRQ(ierr); } ierr = MatDestroy(&Credundant);CHKERRQ(ierr); /* Test MatCreateRedundantMatrix() with user-provided subcomm */ { PetscSubcomm psubcomm; ierr = PetscSubcommCreate(PETSC_COMM_WORLD,&psubcomm);CHKERRQ(ierr); ierr = PetscSubcommSetNumber(psubcomm,nsubcomms);CHKERRQ(ierr); ierr = PetscSubcommSetType(psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr); /* enable runtime switch of psubcomm type, e.g., '-psubcomm_type interlaced */ ierr = PetscSubcommSetFromOptions(psubcomm);CHKERRQ(ierr); ierr = MatCreateRedundantMatrix(C,nsubcomms,PetscSubcommChild(psubcomm),MAT_INITIAL_MATRIX,&Credundant);CHKERRQ(ierr); ierr = MatCreateRedundantMatrix(C,nsubcomms,PetscSubcommChild(psubcomm),MAT_REUSE_MATRIX,&Credundant);CHKERRQ(ierr); ierr = PetscSubcommDestroy(&psubcomm);CHKERRQ(ierr); ierr = MatDestroy(&Credundant);CHKERRQ(ierr); } ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = VecDestroy(&y);CHKERRQ(ierr); ierr = MatDestroy(&C);CHKERRQ(ierr); ierr = PetscFinalize(); return ierr; }
int main(int argc,char **args) { KSP subksp; Mat A,subA; Vec x,b,u,subb,subx,subu; PetscViewer fd; char file[PETSC_MAX_PATH_LEN]; PetscBool flg; PetscErrorCode ierr; PetscInt i,m,n,its; PetscReal norm; PetscMPIInt rank,size; MPI_Comm comm,subcomm; PetscSubcomm psubcomm; PetscInt nsubcomm=1,id; PetscScalar *barray,*xarray,*uarray,*array,one=1.0; PetscInt type=1; PetscInitialize(&argc,&args,(char*)0,help); /* Load the matrix */ ierr = PetscOptionsGetString(NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr); if (!flg) SETERRQ(PETSC_COMM_WORLD,1,"Must indicate binary file with the -f option"); ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr); /* Load the matrix; then destroy the viewer.*/ ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); ierr = MatLoad(A,fd);CHKERRQ(ierr); ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr); ierr = PetscObjectGetComm((PetscObject)A,&comm);CHKERRQ(ierr); ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr); ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr); /* Create rhs vector b */ ierr = MatGetLocalSize(A,&m,NULL);CHKERRQ(ierr); ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr); ierr = VecSetSizes(b,m,PETSC_DECIDE);CHKERRQ(ierr); ierr = VecSetFromOptions(b);CHKERRQ(ierr); ierr = VecSet(b,one);CHKERRQ(ierr); ierr = VecDuplicate(b,&x);CHKERRQ(ierr); ierr = VecDuplicate(b,&u);CHKERRQ(ierr); ierr = VecSet(x,0.0);CHKERRQ(ierr); /* Test MatGetMultiProcBlock() */ ierr = PetscOptionsGetInt(NULL,"-nsubcomm",&nsubcomm,NULL);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,"-subcomm_type",&type,NULL);CHKERRQ(ierr); ierr = PetscSubcommCreate(comm,&psubcomm);CHKERRQ(ierr); ierr = PetscSubcommSetNumber(psubcomm,nsubcomm);CHKERRQ(ierr); if (type == PETSC_SUBCOMM_GENERAL) { /* user provides color, subrank and duprank */ PetscMPIInt color,subrank,duprank,subsize; duprank = size-1 - rank; subsize = size/nsubcomm; if (subsize*nsubcomm != size) SETERRQ2(comm,PETSC_ERR_SUP,"This example requires nsubcomm %D divides size %D",nsubcomm,size); color = duprank/subsize; subrank = duprank - color*subsize; ierr = PetscSubcommSetTypeGeneral(psubcomm,color,subrank);CHKERRQ(ierr); } else if (type == PETSC_SUBCOMM_CONTIGUOUS) { ierr = PetscSubcommSetType(psubcomm,PETSC_SUBCOMM_CONTIGUOUS);CHKERRQ(ierr); } else if (type == PETSC_SUBCOMM_INTERLACED) { ierr = PetscSubcommSetType(psubcomm,PETSC_SUBCOMM_INTERLACED);CHKERRQ(ierr); } else SETERRQ1(psubcomm->parent,PETSC_ERR_SUP,"PetscSubcommType %D is not supported yet",type); ierr = PetscSubcommSetFromOptions(psubcomm);CHKERRQ(ierr); subcomm = psubcomm->comm; /* Test MatGetRedundantMatrix() */ if (size > 1) { ierr = MatGetRedundantMatrix(A,nsubcomm,subcomm,MAT_INITIAL_MATRIX,&subA);CHKERRQ(ierr); ierr = MatGetRedundantMatrix(A,nsubcomm,subcomm,MAT_REUSE_MATRIX,&subA);CHKERRQ(ierr); ierr = MatDestroy(&subA);CHKERRQ(ierr); } /* Create subA */ ierr = MatGetMultiProcBlock(A,subcomm,MAT_INITIAL_MATRIX,&subA);CHKERRQ(ierr); ierr = MatGetMultiProcBlock(A,subcomm,MAT_REUSE_MATRIX,&subA);CHKERRQ(ierr); /* Create sub vectors without arrays. Place b's and x's local arrays into subb and subx */ ierr = MatGetLocalSize(subA,&m,&n);CHKERRQ(ierr); ierr = VecCreateMPIWithArray(subcomm,1,m,PETSC_DECIDE,NULL,&subb);CHKERRQ(ierr); ierr = VecCreateMPIWithArray(subcomm,1,n,PETSC_DECIDE,NULL,&subx);CHKERRQ(ierr); ierr = VecCreateMPIWithArray(subcomm,1,n,PETSC_DECIDE,NULL,&subu);CHKERRQ(ierr); ierr = VecGetArray(b,&barray);CHKERRQ(ierr); ierr = VecGetArray(x,&xarray);CHKERRQ(ierr); ierr = VecGetArray(u,&uarray);CHKERRQ(ierr); ierr = VecPlaceArray(subb,barray);CHKERRQ(ierr); ierr = VecPlaceArray(subx,xarray);CHKERRQ(ierr); ierr = VecPlaceArray(subu,uarray);CHKERRQ(ierr); /* Create linear solvers associated with subA */ ierr = KSPCreate(subcomm,&subksp);CHKERRQ(ierr); ierr = KSPSetOperators(subksp,subA,subA);CHKERRQ(ierr); ierr = KSPSetFromOptions(subksp);CHKERRQ(ierr); /* Solve sub systems */ ierr = KSPSolve(subksp,subb,subx);CHKERRQ(ierr); ierr = KSPGetIterationNumber(subksp,&its);CHKERRQ(ierr); /* check residual */ ierr = MatMult(subA,subx,subu);CHKERRQ(ierr); ierr = VecAXPY(subu,-1.0,subb);CHKERRQ(ierr); ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr); if (norm > 1.e-4 && !rank) { ierr = PetscPrintf(PETSC_COMM_WORLD,"[%D] Number of iterations = %3D\n",rank,its);CHKERRQ(ierr); printf("Error: Residual norm of each block |subb - subA*subx |= %g\n",(double)norm); } ierr = VecResetArray(subb);CHKERRQ(ierr); ierr = VecResetArray(subx);CHKERRQ(ierr); ierr = VecResetArray(subu);CHKERRQ(ierr); ierr = PetscOptionsGetInt(NULL,"-subvec_view",&id,&flg);CHKERRQ(ierr); if (flg && rank == id) { ierr = PetscPrintf(PETSC_COMM_SELF,"[%D] subb:\n", rank); ierr = VecGetArray(subb,&array);CHKERRQ(ierr); for (i=0; i<m; i++) printf("%g\n",(double)PetscRealPart(array[i])); ierr = VecRestoreArray(subb,&array);CHKERRQ(ierr); ierr = PetscPrintf(PETSC_COMM_SELF,"[%D] subx:\n", rank); ierr = VecGetArray(subx,&array);CHKERRQ(ierr); for (i=0; i<m; i++) printf("%g\n",(double)PetscRealPart(array[i])); ierr = VecRestoreArray(subx,&array);CHKERRQ(ierr); } ierr = VecRestoreArray(x,&xarray);CHKERRQ(ierr); ierr = VecRestoreArray(b,&barray);CHKERRQ(ierr); ierr = VecRestoreArray(u,&uarray);CHKERRQ(ierr); ierr = MatDestroy(&subA);CHKERRQ(ierr); ierr = VecDestroy(&subb);CHKERRQ(ierr); ierr = VecDestroy(&subx);CHKERRQ(ierr); ierr = VecDestroy(&subu);CHKERRQ(ierr); ierr = KSPDestroy(&subksp);CHKERRQ(ierr); ierr = PetscSubcommDestroy(&psubcomm);CHKERRQ(ierr); ierr = MatDestroy(&A);CHKERRQ(ierr); ierr = VecDestroy(&b);CHKERRQ(ierr); ierr = VecDestroy(&u);CHKERRQ(ierr); ierr = VecDestroy(&x);CHKERRQ(ierr); ierr = PetscFinalize(); return 0; }