PetscErrorCode SNESComputeJacobianDefaultColor(SNES snes,Vec x1,Mat *J,Mat *B,MatStructure *flag,void *ctx)
{
MatFDColoring color = (MatFDColoring)ctx;
PetscErrorCode ierr;
DM dm;
PetscErrorCode (*func)(SNES,Vec,Vec,void*);
Vec F;
void *funcctx;
ISColoring iscoloring;
PetscBool hascolor;
PetscBool solvec;
PetscFunctionBegin;
if (color) PetscValidHeaderSpecific(color,MAT_FDCOLORING_CLASSID,6);
else {ierr = PetscObjectQuery((PetscObject)*B,"SNESMatFDColoring",(PetscObject*)&color);CHKERRQ(ierr);}
*flag = SAME_NONZERO_PATTERN;
ierr = SNESGetFunction(snes,&F,&func,&funcctx);CHKERRQ(ierr);
if (!color) {
ierr = SNESGetDM(snes,&dm);CHKERRQ(ierr);
ierr = DMHasColoring(dm,&hascolor);CHKERRQ(ierr);
if (hascolor) {
ierr = DMCreateColoring(dm,IS_COLORING_GLOBAL,&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringCreate(*B,iscoloring,&color);CHKERRQ(ierr);
ierr = ISColoringDestroy(&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringSetFunction(color,(PetscErrorCode (*)(void))func,funcctx);CHKERRQ(ierr);
ierr = MatFDColoringSetFromOptions(color);CHKERRQ(ierr);
} else {
ierr = MatGetColoring(*B,MATCOLORINGSL,&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringCreate(*B,iscoloring,&color);CHKERRQ(ierr);
ierr = ISColoringDestroy(&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringSetFunction(color,(PetscErrorCode (*)(void))func,(void*)funcctx);CHKERRQ(ierr);
ierr = MatFDColoringSetFromOptions(color);CHKERRQ(ierr);
}
ierr = PetscObjectCompose((PetscObject)*B,"SNESMatFDColoring",(PetscObject)color);CHKERRQ(ierr);
ierr = PetscObjectDereference((PetscObject)color);CHKERRQ(ierr);
}
/* F is only usable if there is no RHS on the SNES and the full solution corresponds to x1 */
ierr = VecEqual(x1,snes->vec_sol,&solvec);CHKERRQ(ierr);
if (!snes->vec_rhs && solvec) {
ierr = MatFDColoringSetF(color,F);CHKERRQ(ierr);
}
ierr = MatFDColoringApply(*B,color,x1,flag,snes);CHKERRQ(ierr);
if (*J != *B) {
ierr = MatAssemblyBegin(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*J,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat A,Asp;
PetscViewer fd; /* viewer */
char file[PETSC_MAX_PATH_LEN]; /* input file name */
PetscErrorCode ierr;
PetscInt m,n,rstart,rend;
PetscBool flg;
PetscInt row,ncols,j,nrows,nnzA=0,nnzAsp=0;
const PetscInt *cols;
const PetscScalar *vals;
PetscReal norm,percent,val,dtol=1.e-16;
PetscMPIInt rank;
MatInfo matinfo;
PetscInt Dnnz,Onnz;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* Determine files from which we read the linear systems. */
ierr = PetscOptionsGetString(PETSC_NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_WORLD,1,"Must indicate binary file with the -f option");
/* Open binary file. Note that we use FILE_MODE_READ to indicate
reading from this file. */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
/* Load the matrix; then destroy the viewer. */
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetOptionsPrefix(A,"a_");CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
ierr = MatGetSize(A,&m,&n);CHKERRQ(ierr);
ierr = MatGetInfo(A,MAT_LOCAL,&matinfo);CHKERRQ(ierr);
//printf("matinfo.nz_used %g\n",matinfo.nz_used);
/* Get a sparse matrix Asp by dumping zero entries of A */
ierr = MatCreate(PETSC_COMM_WORLD,&Asp);CHKERRQ(ierr);
ierr = MatSetSizes(Asp,m,n,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetOptionsPrefix(Asp,"asp_");CHKERRQ(ierr);
ierr = MatSetFromOptions(Asp);CHKERRQ(ierr);
Dnnz = (PetscInt)matinfo.nz_used/m + 1;
Onnz = Dnnz/2;
printf("Dnnz %d %d\n",Dnnz,Onnz);
ierr = MatSeqAIJSetPreallocation(Asp,Dnnz,PETSC_NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(Asp,Dnnz,PETSC_NULL,Onnz,PETSC_NULL);CHKERRQ(ierr);
/* Check zero rows */
ierr = MatGetOwnershipRange(A,&rstart,&rend);CHKERRQ(ierr);
nrows = 0;
for (row=rstart; row<rend; row++){
ierr = MatGetRow(A,row,&ncols,&cols,&vals);CHKERRQ(ierr);
nnzA += ncols;
norm = 0.0;
for (j=0; j<ncols; j++){
val = PetscAbsScalar(vals[j]);
if (norm < val) norm = norm;
if (val > dtol){
ierr = MatSetValues(Asp,1,&row,1,&cols[j],&vals[j],INSERT_VALUES);CHKERRQ(ierr);
nnzAsp++;
}
}
if (!norm) nrows++;
ierr = MatRestoreRow(A,row,&ncols,&cols,&vals);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(Asp,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Asp,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
percent=(PetscReal)nnzA*100/(m*n);
ierr = PetscPrintf(PETSC_COMM_SELF," [%d] Matrix A local size %d,%d; nnzA %d, %g percent; No. of zero rows: %d\n",rank,m,n,nnzA,percent,nrows);
percent=(PetscReal)nnzAsp*100/(m*n);
ierr = PetscPrintf(PETSC_COMM_SELF," [%d] Matrix Asp nnzAsp %d, %g percent\n",rank,nnzAsp,percent);
/* investigate matcoloring for Asp */
PetscBool Asp_coloring = PETSC_FALSE;
ierr = PetscOptionsHasName(PETSC_NULL,"-Asp_color",&Asp_coloring);CHKERRQ(ierr);
if (Asp_coloring){
ISColoring iscoloring;
MatFDColoring matfdcoloring;
ierr = PetscPrintf(PETSC_COMM_WORLD," Create coloring of Asp...\n");
ierr = MatGetColoring(Asp,MATCOLORINGSL,&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringCreate(Asp,iscoloring,&matfdcoloring);CHKERRQ(ierr);
ierr = MatFDColoringSetFromOptions(matfdcoloring);CHKERRQ(ierr);
//ierr = MatFDColoringView(matfdcoloring,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = ISColoringDestroy(&iscoloring);CHKERRQ(ierr);
ierr = MatFDColoringDestroy(&matfdcoloring);CHKERRQ(ierr);
}
/* Write Asp in binary for study - see ~petsc/src/mat/examples/tests/ex124.c */
PetscBool Asp_write = PETSC_FALSE;
ierr = PetscOptionsHasName(PETSC_NULL,"-Asp_write",&Asp_write);CHKERRQ(ierr);
if (Asp_write){
PetscViewer viewer;
ierr = PetscPrintf(PETSC_COMM_SELF,"Write Asp into file Asp.dat ...\n");
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,"Asp.dat",FILE_MODE_WRITE,&viewer);CHKERRQ(ierr);
ierr = MatView(Asp,viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&Asp);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
void
NonlinearSystem::setupColoringFiniteDifferencedPreconditioner()
{
#ifdef LIBMESH_HAVE_PETSC
// Make sure that libMesh isn't going to override our preconditioner
_transient_sys.nonlinear_solver->jacobian = nullptr;
PetscNonlinearSolver<Number> & petsc_nonlinear_solver =
dynamic_cast<PetscNonlinearSolver<Number> &>(*_transient_sys.nonlinear_solver);
// Pointer to underlying PetscMatrix type
PetscMatrix<Number> * petsc_mat = dynamic_cast<PetscMatrix<Number> *>(_transient_sys.matrix);
#if PETSC_VERSION_LESS_THAN(3, 2, 0)
// This variable is only needed for PETSC < 3.2.0
PetscVector<Number> * petsc_vec =
dynamic_cast<PetscVector<Number> *>(_transient_sys.solution.get());
#endif
Moose::compute_jacobian(*_transient_sys.current_local_solution, *petsc_mat, _transient_sys);
if (!petsc_mat)
mooseError("Could not convert to Petsc matrix.");
petsc_mat->close();
PetscErrorCode ierr = 0;
ISColoring iscoloring;
#if PETSC_VERSION_LESS_THAN(3, 2, 0)
// PETSc 3.2.x
ierr = MatGetColoring(petsc_mat->mat(), MATCOLORING_LF, &iscoloring);
CHKERRABORT(libMesh::COMM_WORLD, ierr);
#elif PETSC_VERSION_LESS_THAN(3, 5, 0)
// PETSc 3.3.x, 3.4.x
ierr = MatGetColoring(petsc_mat->mat(), MATCOLORINGLF, &iscoloring);
CHKERRABORT(_communicator.get(), ierr);
#else
// PETSc 3.5.x
MatColoring matcoloring;
ierr = MatColoringCreate(petsc_mat->mat(), &matcoloring);
CHKERRABORT(_communicator.get(), ierr);
ierr = MatColoringSetType(matcoloring, MATCOLORINGLF);
CHKERRABORT(_communicator.get(), ierr);
ierr = MatColoringSetFromOptions(matcoloring);
CHKERRABORT(_communicator.get(), ierr);
ierr = MatColoringApply(matcoloring, &iscoloring);
CHKERRABORT(_communicator.get(), ierr);
ierr = MatColoringDestroy(&matcoloring);
CHKERRABORT(_communicator.get(), ierr);
#endif
MatFDColoringCreate(petsc_mat->mat(), iscoloring, &_fdcoloring);
MatFDColoringSetFromOptions(_fdcoloring);
MatFDColoringSetFunction(_fdcoloring,
(PetscErrorCode(*)(void)) & libMesh::__libmesh_petsc_snes_fd_residual,
&petsc_nonlinear_solver);
#if !PETSC_RELEASE_LESS_THAN(3, 5, 0)
MatFDColoringSetUp(petsc_mat->mat(), iscoloring, _fdcoloring);
#endif
#if PETSC_VERSION_LESS_THAN(3, 4, 0)
SNESSetJacobian(petsc_nonlinear_solver.snes(),
petsc_mat->mat(),
petsc_mat->mat(),
SNESDefaultComputeJacobianColor,
_fdcoloring);
#else
SNESSetJacobian(petsc_nonlinear_solver.snes(),
petsc_mat->mat(),
petsc_mat->mat(),
SNESComputeJacobianDefaultColor,
_fdcoloring);
#endif
#if PETSC_VERSION_LESS_THAN(3, 2, 0)
Mat my_mat = petsc_mat->mat();
MatStructure my_struct;
SNESComputeJacobian(
petsc_nonlinear_solver.snes(), petsc_vec->vec(), &my_mat, &my_mat, &my_struct);
#endif
#if PETSC_VERSION_LESS_THAN(3, 2, 0)
ISColoringDestroy(iscoloring);
#else
// PETSc 3.3.0
ISColoringDestroy(&iscoloring);
#endif
#endif
}
PetscErrorCode MatRARtSymbolic_SeqAIJ_SeqAIJ(Mat A,Mat R,PetscReal fill,Mat *C)
{
PetscErrorCode ierr;
Mat P;
PetscInt *rti,*rtj;
Mat_RARt *rart;
PetscContainer container;
MatTransposeColoring matcoloring;
ISColoring iscoloring;
Mat Rt_dense,RARt_dense;
PetscLogDouble GColor=0.0,MCCreate=0.0,MDenCreate=0.0,t0,tf,etime=0.0;
Mat_SeqAIJ *c;
PetscFunctionBegin;
ierr = PetscGetTime(&t0);CHKERRQ(ierr);
/* create symbolic P=Rt */
ierr = MatGetSymbolicTranspose_SeqAIJ(R,&rti,&rtj);CHKERRQ(ierr);
ierr = MatCreateSeqAIJWithArrays(PETSC_COMM_SELF,R->cmap->n,R->rmap->n,rti,rtj,PETSC_NULL,&P);CHKERRQ(ierr);
/* get symbolic C=Pt*A*P */
ierr = MatPtAPSymbolic_SeqAIJ_SeqAIJ(A,P,fill,C);CHKERRQ(ierr);
(*C)->rmap->bs = R->rmap->bs;
(*C)->cmap->bs = R->rmap->bs;
/* create a supporting struct */
ierr = PetscNew(Mat_RARt,&rart);CHKERRQ(ierr);
/* attach the supporting struct to C */
ierr = PetscContainerCreate(PETSC_COMM_SELF,&container);CHKERRQ(ierr);
ierr = PetscContainerSetPointer(container,rart);CHKERRQ(ierr);
ierr = PetscContainerSetUserDestroy(container,PetscContainerDestroy_Mat_RARt);CHKERRQ(ierr);
ierr = PetscObjectCompose((PetscObject)(*C),"Mat_RARt",(PetscObject)container);CHKERRQ(ierr);
ierr = PetscContainerDestroy(&container);CHKERRQ(ierr);
ierr = PetscGetTime(&tf);CHKERRQ(ierr);
etime += tf - t0;
/* Create MatTransposeColoring from symbolic C=R*A*R^T */
c=(Mat_SeqAIJ*)(*C)->data;
ierr = PetscGetTime(&t0);CHKERRQ(ierr);
ierr = MatGetColoring(*C,MATCOLORINGLF,&iscoloring);CHKERRQ(ierr);
ierr = PetscGetTime(&tf);CHKERRQ(ierr);
GColor += tf - t0;
ierr = PetscGetTime(&t0);CHKERRQ(ierr);
ierr = MatTransposeColoringCreate(*C,iscoloring,&matcoloring);CHKERRQ(ierr);
rart->matcoloring = matcoloring;
ierr = ISColoringDestroy(&iscoloring);CHKERRQ(ierr);
ierr = PetscGetTime(&tf);CHKERRQ(ierr);
MCCreate += tf - t0;
ierr = PetscGetTime(&t0);CHKERRQ(ierr);
/* Create Rt_dense */
ierr = MatCreate(PETSC_COMM_SELF,&Rt_dense);CHKERRQ(ierr);
ierr = MatSetSizes(Rt_dense,A->cmap->n,matcoloring->ncolors,A->cmap->n,matcoloring->ncolors);CHKERRQ(ierr);
ierr = MatSetType(Rt_dense,MATSEQDENSE);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(Rt_dense,PETSC_NULL);CHKERRQ(ierr);
Rt_dense->assembled = PETSC_TRUE;
rart->Rt = Rt_dense;
/* Create RARt_dense = R*A*Rt_dense */
ierr = MatCreate(PETSC_COMM_SELF,&RARt_dense);CHKERRQ(ierr);
ierr = MatSetSizes(RARt_dense,(*C)->rmap->n,matcoloring->ncolors,(*C)->rmap->n,matcoloring->ncolors);CHKERRQ(ierr);
ierr = MatSetType(RARt_dense,MATSEQDENSE);CHKERRQ(ierr);
ierr = MatSeqDenseSetPreallocation(RARt_dense,PETSC_NULL);CHKERRQ(ierr);
rart->RARt = RARt_dense;
/* Allocate work array to store columns of A*R^T used in MatMatMatMultNumeric_SeqAIJ_SeqAIJ_SeqDense() */
ierr = PetscMalloc(A->rmap->n*4*sizeof(PetscScalar),&rart->work);CHKERRQ(ierr);
ierr = PetscGetTime(&tf);CHKERRQ(ierr);
MDenCreate += tf - t0;
rart->destroy = (*C)->ops->destroy;
(*C)->ops->destroy = MatDestroy_SeqAIJ_RARt;
/* clean up */
ierr = MatRestoreSymbolicTranspose_SeqAIJ(R,&rti,&rtj);CHKERRQ(ierr);
ierr = MatDestroy(&P);CHKERRQ(ierr);
#if defined(PETSC_USE_INFO)
{
PetscReal density= (PetscReal)(c->nz)/(RARt_dense->rmap->n*RARt_dense->cmap->n);
ierr = PetscInfo6(*C,"RARt_den %D %D; Rt_den %D %D, (RARt->nz %D)/(m*ncolors)=%g\n",RARt_dense->rmap->n,RARt_dense->cmap->n,Rt_dense->rmap->n,Rt_dense->cmap->n,c->nz,density);CHKERRQ(ierr);
ierr = PetscInfo5(*C,"Sym = GetColor %g + MColorCreate %g + MDenCreate %g + other %g = %g\n",GColor,MCCreate,MDenCreate,etime,GColor+MCCreate+MDenCreate+etime);CHKERRQ(ierr);
}
#endif
PetscFunctionReturn(0);
}
