PetscErrorCode MatView_LMVM(Mat B, PetscViewer pv)
{
Mat_LMVM *lmvm = (Mat_LMVM*)B->data;
PetscErrorCode ierr;
PetscBool isascii;
MatType type;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)pv,PETSCVIEWERASCII,&isascii);CHKERRQ(ierr);
if (isascii) {
ierr = MatGetType(B, &type);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pv,"Max. storage: %D\n",lmvm->m);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pv,"Used storage: %D\n",lmvm->k+1);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pv,"Number of updates: %D\n",lmvm->nupdates);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pv,"Number of rejects: %D\n",lmvm->nrejects);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(pv,"Number of resets: %D\n",lmvm->nresets);CHKERRQ(ierr);
if (lmvm->J0) {
ierr = PetscViewerASCIIPrintf(pv,"J0 Matrix:\n");CHKERRQ(ierr);
ierr = PetscViewerPushFormat(pv, PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr);
ierr = MatView(lmvm->J0, pv);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(pv);CHKERRQ(ierr);
}
}
PetscFunctionReturn(0);
}
PetscErrorCode DumpCSR(Mat A,PetscInt shift,PetscBool symmetric,PetscBool compressed)
{
MatType type;
PetscInt i,j,nr,bs = 1;
const PetscInt *ia,*ja;
PetscBool done,isseqbaij,isseqsbaij;
PetscErrorCode ierr;
PetscFunctionBeginUser;
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQBAIJ,&isseqbaij);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQSBAIJ,&isseqsbaij);CHKERRQ(ierr);
if (isseqbaij || isseqsbaij) {
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
}
ierr = MatGetType(A,&type);CHKERRQ(ierr);
ierr = MatGetRowIJ(A,shift,symmetric,compressed,&nr,&ia,&ja,&done);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"===========================================================\n");CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_SELF,"CSR for %s: shift %D symmetric %D compressed %D\n",type,shift,(PetscInt)symmetric,(PetscInt)compressed);CHKERRQ(ierr);
for (i=0;i<nr;i++) {
ierr = PetscPrintf(PETSC_COMM_SELF,"%D:",i+shift);CHKERRQ(ierr);
for (j=ia[i];j<ia[i+1];j++) {
ierr = PetscPrintf(PETSC_COMM_SELF," %D",ja[j-shift]);CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_SELF,"\n");CHKERRQ(ierr);
}
ierr = MatRestoreRowIJ(A,shift,symmetric,compressed,&nr,&ia,&ja,&done);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static PetscErrorCode MatDuplicate_LMVM(Mat B, MatDuplicateOption op, Mat *mat)
{
Mat_LMVM *bctx = (Mat_LMVM*)B->data;
Mat_LMVM *mctx;
PetscErrorCode ierr;
MatType lmvmType;
Mat A;
PetscFunctionBegin;
ierr = MatGetType(B, &lmvmType);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)B), mat);CHKERRQ(ierr);
ierr = MatSetType(*mat, lmvmType);CHKERRQ(ierr);
A = *mat;
mctx = (Mat_LMVM*)A->data;
mctx->m = bctx->m;
mctx->ksp_max_it = bctx->ksp_max_it;
mctx->ksp_rtol = bctx->ksp_rtol;
mctx->ksp_atol = bctx->ksp_atol;
mctx->shift = bctx->shift;
ierr = KSPSetTolerances(mctx->J0ksp, mctx->ksp_rtol, mctx->ksp_atol, PETSC_DEFAULT, mctx->ksp_max_it);CHKERRQ(ierr);
ierr = MatLMVMAllocate(*mat, bctx->Xprev, bctx->Fprev);CHKERRQ(ierr);
if (op == MAT_COPY_VALUES) {
ierr = MatCopy(B, *mat, SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
PetscErrorCode CheckMat(Mat A, Mat B, PetscBool usemult, const char* func)
{
Mat Bcheck;
PetscReal error;
PetscErrorCode ierr;
PetscFunctionBeginUser;
if (!usemult) {
if (B) {
MatType Btype;
ierr = MatGetType(B,&Btype);CHKERRQ(ierr);
ierr = MatConvert(A,Btype,MAT_INITIAL_MATRIX,&Bcheck);CHKERRQ(ierr);
} else {
ierr = MatConvert(A,MATAIJ,MAT_INITIAL_MATRIX,&Bcheck);CHKERRQ(ierr);
}
if (B) { /* if B is present, subtract it */
ierr = MatAXPY(Bcheck,-1.,B,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
}
ierr = MatNorm(Bcheck,NORM_INFINITY,&error);CHKERRQ(ierr);
if (error > PETSC_SQRT_MACHINE_EPSILON) {
ISLocalToGlobalMapping rl2g,cl2g;
ierr = PetscObjectSetName((PetscObject)Bcheck,"Assembled Bcheck");CHKERRQ(ierr);
ierr = MatView(Bcheck,NULL);CHKERRQ(ierr);
if (B) {
ierr = PetscObjectSetName((PetscObject)B,"Assembled AIJ");CHKERRQ(ierr);
ierr = MatView(B,NULL);CHKERRQ(ierr);
ierr = MatDestroy(&Bcheck);CHKERRQ(ierr);
ierr = MatConvert(A,MATAIJ,MAT_INITIAL_MATRIX,&Bcheck);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)Bcheck,"Assembled IS");CHKERRQ(ierr);
ierr = MatView(Bcheck,NULL);CHKERRQ(ierr);
}
ierr = MatDestroy(&Bcheck);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject)A,"MatIS");CHKERRQ(ierr);
ierr = MatView(A,NULL);CHKERRQ(ierr);
ierr = MatGetLocalToGlobalMapping(A,&rl2g,&cl2g);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(rl2g,NULL);CHKERRQ(ierr);
ierr = ISLocalToGlobalMappingView(cl2g,NULL);CHKERRQ(ierr);
SETERRQ2(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"ERROR ON %s: %g",func,error);
}
ierr = MatDestroy(&Bcheck);CHKERRQ(ierr);
} else {
PetscBool ok,okt;
ierr = MatMultEqual(A,B,3,&ok);CHKERRQ(ierr);
ierr = MatMultTransposeEqual(A,B,3,&okt);CHKERRQ(ierr);
if (!ok || !okt) SETERRQ3(PETSC_COMM_WORLD,PETSC_ERR_PLIB,"ERROR ON %s: mult ok ? %d, multtranspose ok ? %d",func,ok,okt);
}
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Mat C=0;
PetscScalar v;
PetscInt Ii,J,Istart,Iend;
PetscErrorCode ierr;
PetscInt i,j,m = 3,n = 2;
PetscMPIInt size,rank;
PetscInt solve_count;
const MatType type;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = PetscOptionsGetInt(PETSC_NULL,"-m",&m,PETSC_NULL);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
n = 2*size;
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,m*n,m*n);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
for (solve_count=0; solve_count<3; solve_count++){
if (solve_count == 1){
ierr = MatSetType(C,MATSBAIJ);CHKERRQ(ierr);
ierr = MatSetOption(C,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
} else {
ierr = MatSetType(C,MATMPIDENSE);CHKERRQ(ierr);
}
ierr = MatGetOwnershipRange(C,&Istart,&Iend);CHKERRQ(ierr);
for (Ii=Istart; Ii<Iend; Ii++) {
v = -1.0; i = Ii/n; j = Ii - i*n;
if (i>0) {J = Ii - n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (i<m-1) {J = Ii + n; ierr = MatSetValues(C,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j>0) {J = Ii - 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
if (j<n-1) {J = Ii + 1; ierr = MatSetValues(C,1,&Ii,1,&J,&v,ADD_VALUES);CHKERRQ(ierr);}
v = 4.0; ierr = MatSetValues(C,1,&Ii,1,&Ii,&v,ADD_VALUES);
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatGetType(C,&type);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," case %D, matrix type: %s\n",solve_count,type);
}
/* Free work space. */
ierr = MatDestroy(C);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
void
create(Mat new_mat) {
mat = new_mat;
//For this code to work, we're going to need the matrix structure for the sequential portion of this matrix.
//Therefore, extract a sequential AIJ matrix.
MatType type;
MatGetType(mat, &type);
assert(!strcmp(type, MATSEQAIJ));
PetscTruth done = PETSC_FALSE;
MatGetRowIJ(mat, 0, PETSC_FALSE, PETSC_FALSE, &nrows, &ia, &ja, &done);
assert(done == PETSC_TRUE || "Unexpected error: can't get csr structure from matrix");
MatGetArray(mat, &data);
valid = true;
}
RawGraph(Mat new_global_mat) {
global_mat = new_global_mat;
//For this code to work, we're going to need the matrix structure for the sequential portion of this matrix.
//Therefore, extract a sequential AIJ matrix.
MatType type;
MatGetType(global_mat, &type);
if (!strcmp(type,MATSEQAIJ)) {
MatDuplicate(global_mat, MAT_DO_NOT_COPY_VALUES, &local_mat);
} else {
MatGetLocalMat(global_mat, MAT_INITIAL_MATRIX, &local_mat);
}
MatZeroEntries(local_mat);
MatGetOwnershipRange(global_mat, &row_begin, &row_end);
//MatView(local_mat, PETSC_VIEWER_DRAW_SELF);
seq_raw.create(local_mat);
}
int TaoPetscApplication::GetLinearSolver(TaoMat *tH, int stype, TaoLinearSolver **SS)
{
TaoMatPetsc *pH = dynamic_cast <TaoMatPetsc *> (tH);
Mat pm, pmpre;
MatStructure pflag;
MPI_Comm comm2;
int info;
const MatType mtype;
PC pc;
KSP newksp;
TaoLinearSolverPetsc *S;
PetscTruth flg1,flg2;
PetscFunctionBegin;
info = pH->GetMatrix(&pm,&pmpre,&pflag); CHKERRQ(info);
info = TaoAppGetKSP(papp,&newksp); CHKERRQ(info);
if (this->ksptmp==0){
info = TaoWrapKSP( newksp, &S ); CHKERRQ(info);
this->ksptmp=S;
}
info=this->ksptmp->SetKSP(newksp);CHKERRQ(info);
*SS=this->ksptmp;
if (pm) {
info = PetscObjectGetComm((PetscObject)pm,&comm2); CHKERRQ(info);
info = MatGetType(pm,&mtype); CHKERRQ(info);
info = PetscStrncmp(mtype,MATSEQDENSE,10,&flg1); CHKERRQ(info);
info = PetscStrncmp(mtype,MATMPIDENSE,10,&flg2); CHKERRQ(info);
}
else {
comm2 = this->comm;
}
info = KSPGetPC(newksp,&pc); CHKERRQ(info);
info=KSPSetFromOptions(newksp); CHKERRQ(info);
PetscFunctionReturn(0);
}
/*
name[] is operator name
*/
PetscErrorCode BSSCR_MatInfoLog(PetscViewer v,Mat A,const char name[])
{
MatInfo i;
PetscReal nrm_1,nrm_f,nrm_inf;
MatType mtype;
PetscInt M,N;
PetscViewerType vtype;
PetscTruth isascii;
PetscViewerGetType( v, &vtype );
Stg_PetscObjectTypeCompare( (PetscObject)v,PETSC_VIEWER_ASCII,&isascii );
if (!isascii) { PetscFunctionReturn(0); }
MatGetSize( A, &M,&N );
MatGetInfo(A,MAT_GLOBAL_SUM,&i);
MatGetType( A, &mtype );
MatNorm( A, NORM_1, &nrm_1 );
MatNorm( A, NORM_FROBENIUS, &nrm_f );
MatNorm( A, NORM_INFINITY, &nrm_inf );
PetscViewerASCIIPrintf( v, "MatInfo: %s \n", name );
PetscViewerASCIIPushTab(v);
PetscViewerASCIIPrintf( v, "type=%s \n", mtype );
PetscViewerASCIIPrintf( v, "dimension=%Dx%D \n", M,N );
PetscViewerASCIIPrintf( v, "nnz=%D (total)\n", (PetscInt)i.nz_used );
PetscViewerASCIIPrintf( v, "nnz=%D (allocated)\n", (PetscInt)i.nz_allocated );
PetscViewerASCIIPrintf( v, "|A|_1 = %1.12e\n", nrm_1 );
PetscViewerASCIIPrintf( v, "|A|_frobenius = %1.12e\n", nrm_f );
PetscViewerASCIIPrintf( v, "|A|_inf = %1.12e\n", nrm_inf );
PetscViewerASCIIPopTab(v);
PetscFunctionReturn(0);
}
/*
PCGAMGCreateGraph - create simple scaled scalar graph from matrix
Input Parameter:
. Amat - matrix
Output Parameter:
. a_Gmaat - eoutput scalar graph (symmetric?)
*/
PetscErrorCode PCGAMGCreateGraph(Mat Amat, Mat *a_Gmat)
{
PetscErrorCode ierr;
PetscInt Istart,Iend,Ii,jj,kk,ncols,nloc,NN,MM,bs;
MPI_Comm comm;
Mat Gmat;
MatType mtype;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Amat,&comm);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Amat, &Istart, &Iend);CHKERRQ(ierr);
ierr = MatGetSize(Amat, &MM, &NN);CHKERRQ(ierr);
ierr = MatGetBlockSize(Amat, &bs);CHKERRQ(ierr);
nloc = (Iend-Istart)/bs;
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventBegin(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
if (bs > 1) {
const PetscScalar *vals;
const PetscInt *idx;
PetscInt *d_nnz, *o_nnz,*w0,*w1,*w2;
PetscBool ismpiaij,isseqaij;
/*
Determine the preallocation needed for the scalar matrix derived from the vector matrix.
*/
ierr = PetscObjectBaseTypeCompare((PetscObject)Amat,MATSEQAIJ,&isseqaij);CHKERRQ(ierr);
ierr = PetscObjectBaseTypeCompare((PetscObject)Amat,MATMPIAIJ,&ismpiaij);CHKERRQ(ierr);
ierr = PetscMalloc2(nloc, &d_nnz,isseqaij ? 0 : nloc, &o_nnz);CHKERRQ(ierr);
if (isseqaij) {
PetscInt max_d_nnz;
/*
Determine exact preallocation count for (sequential) scalar matrix
*/
ierr = MatSeqAIJGetMaxRowNonzeros(Amat,&max_d_nnz);CHKERRQ(ierr);
max_d_nnz = PetscMin(nloc,bs*max_d_nnz);CHKERRQ(ierr);
ierr = PetscMalloc3(max_d_nnz, &w0,max_d_nnz, &w1,max_d_nnz, &w2);CHKERRQ(ierr);
for (Ii = 0, jj = 0; Ii < Iend; Ii += bs, jj++) {
ierr = MatCollapseRows(Amat,Ii,bs,w0,w1,w2,&d_nnz[jj],NULL);CHKERRQ(ierr);
}
ierr = PetscFree3(w0,w1,w2);CHKERRQ(ierr);
} else if (ismpiaij) {
Mat Daij,Oaij;
const PetscInt *garray;
PetscInt max_d_nnz;
ierr = MatMPIAIJGetSeqAIJ(Amat,&Daij,&Oaij,&garray);CHKERRQ(ierr);
/*
Determine exact preallocation count for diagonal block portion of scalar matrix
*/
ierr = MatSeqAIJGetMaxRowNonzeros(Daij,&max_d_nnz);CHKERRQ(ierr);
max_d_nnz = PetscMin(nloc,bs*max_d_nnz);CHKERRQ(ierr);
ierr = PetscMalloc3(max_d_nnz, &w0,max_d_nnz, &w1,max_d_nnz, &w2);CHKERRQ(ierr);
for (Ii = 0, jj = 0; Ii < Iend - Istart; Ii += bs, jj++) {
ierr = MatCollapseRows(Daij,Ii,bs,w0,w1,w2,&d_nnz[jj],NULL);CHKERRQ(ierr);
}
ierr = PetscFree3(w0,w1,w2);CHKERRQ(ierr);
/*
Over estimate (usually grossly over), preallocation count for off-diagonal portion of scalar matrix
*/
for (Ii = 0, jj = 0; Ii < Iend - Istart; Ii += bs, jj++) {
o_nnz[jj] = 0;
for (kk=0; kk<bs; kk++) { /* rows that get collapsed to a single row */
ierr = MatGetRow(Oaij,Ii+kk,&ncols,0,0);CHKERRQ(ierr);
o_nnz[jj] += ncols;
ierr = MatRestoreRow(Oaij,Ii+kk,&ncols,0,0);CHKERRQ(ierr);
}
if (o_nnz[jj] > (NN/bs-nloc)) o_nnz[jj] = NN/bs-nloc;
}
} else SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"Require AIJ matrix type");
/* get scalar copy (norms) of matrix */
ierr = MatGetType(Amat,&mtype);CHKERRQ(ierr);
ierr = MatCreate(comm, &Gmat);CHKERRQ(ierr);
ierr = MatSetSizes(Gmat,nloc,nloc,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetBlockSizes(Gmat, 1, 1);CHKERRQ(ierr);
ierr = MatSetType(Gmat, mtype);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(Gmat,0,d_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(Gmat,0,d_nnz,0,o_nnz);CHKERRQ(ierr);
ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr);
for (Ii = Istart; Ii < Iend; Ii++) {
PetscInt dest_row = Ii/bs;
ierr = MatGetRow(Amat,Ii,&ncols,&idx,&vals);CHKERRQ(ierr);
for (jj=0; jj<ncols; jj++) {
PetscInt dest_col = idx[jj]/bs;
PetscScalar sv = PetscAbs(PetscRealPart(vals[jj]));
ierr = MatSetValues(Gmat,1,&dest_row,1,&dest_col,&sv,ADD_VALUES);CHKERRQ(ierr);
}
ierr = MatRestoreRow(Amat,Ii,&ncols,&idx,&vals);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(Gmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Gmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
} else {
/* just copy scalar matrix - abs() not taken here but scaled later */
ierr = MatDuplicate(Amat, MAT_COPY_VALUES, &Gmat);CHKERRQ(ierr);
}
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
*a_Gmat = Gmat;
PetscFunctionReturn(0);
}
/*
Builds
B Y1 X1 Bt
and creates a ksp when C=0, otherwise it builds
B Y1 X1 Bt - C
*/
PetscErrorCode BSSCR_PCScGtKGUseStandardBBtOperator( PC pc )
{
PC_SC_GtKG ctx = (PC_SC_GtKG)pc->data;
PetscReal fill;
Mat diag_mat,C;
Vec diag;
PetscInt M,N, m,n;
MPI_Comm comm;
PetscInt nnz_I, nnz_G;
MatType mtype;
const char *prefix;
Mat BBt;
KSP ksp;
PetscTruth ivalue, flg, has_cnst_nullsp;
BSSCR_BSSCR_pc_error_ScGtKG( pc, __func__ );
/* Assemble BBt */
MatGetSize( ctx->Bt, &M, &N );
MatGetLocalSize( ctx->Bt, &m, &n );
MatGetVecs( ctx->Bt, PETSC_NULL, &diag );
/* Define diagonal matrix Y1 X1 */
VecPointwiseMult( diag, ctx->Y1, ctx->X1 );
PetscObjectGetComm( (PetscObject)ctx->F, &comm );
MatCreate( comm, &diag_mat );
MatSetSizes( diag_mat, m,m , M, M );
#if (((PETSC_VERSION_MAJOR==3) && (PETSC_VERSION_MINOR>=3)) || (PETSC_VERSION_MAJOR>3) )
MatSetUp(diag_mat);
#endif
MatGetType( ctx->Bt, &mtype );
MatSetType( diag_mat, mtype );
MatDiagonalSet( diag_mat, diag, INSERT_VALUES );
/* Build operator B Y1 X1 Bt */
BSSCR_BSSCR_get_number_nonzeros_AIJ_ScGtKG( diag_mat, &nnz_I );
BSSCR_BSSCR_get_number_nonzeros_AIJ_ScGtKG( ctx->Bt, &nnz_G );
/*
Not sure the best way to estimate the fill factor.
BBt is a laplacian on the pressure space.
This might tell us something useful...
*/
fill = (PetscReal)(nnz_G)/(PetscReal)( nnz_I );
MatPtAP( diag_mat, ctx->Bt, MAT_INITIAL_MATRIX, fill, &BBt );
Stg_MatDestroy(&diag_mat );
Stg_VecDestroy(&diag );
C = ctx->C;
if( C !=PETSC_NULL ) {
MatAXPY( BBt, -1.0, C, DIFFERENT_NONZERO_PATTERN );
}
/* Build the solver */
KSPCreate( ((PetscObject)pc)->comm, &ksp );
Stg_KSPSetOperators( ksp, BBt, BBt, SAME_NONZERO_PATTERN );
PCGetOptionsPrefix( pc,&prefix );
KSPSetOptionsPrefix( ksp, prefix );
KSPAppendOptionsPrefix( ksp, "pc_gtkg_" ); /* -pc_GtKG_ksp_type <type>, -ksp_GtKG_pc_type <type> */
BSSCR_PCScGtKGSetKSP( pc, ksp );
BSSCR_MatContainsConstNullSpace( BBt, NULL, &has_cnst_nullsp );
if( has_cnst_nullsp == PETSC_TRUE ) {
BSSCR_PCScGtKGAttachNullSpace( pc );
}
PetscOptionsGetTruth( PETSC_NULL, "-pc_gtkg_monitor", &ivalue, &flg );
BSSCR_PCScGtKGSetSubKSPMonitor( pc, ivalue );
Stg_KSPDestroy(&ksp);
Stg_MatDestroy(&BBt);
PetscFunctionReturn(0);
}
/*@C
PCGAMGFilterGraph - filter (remove zero and possibly small values from the) graph and make it symmetric if requested
Collective on Mat
Input Parameter:
+ a_Gmat - the graph
. vfilter - threshold paramter [0,1)
- symm - make the result symmetric
Level: developer
Notes:
This is called before graph coarsers are called.
.seealso: PCGAMGSetThreshold()
@*/
PetscErrorCode PCGAMGFilterGraph(Mat *a_Gmat,PetscReal vfilter,PetscBool symm)
{
PetscErrorCode ierr;
PetscInt Istart,Iend,Ii,jj,ncols,nnz0,nnz1, NN, MM, nloc;
PetscMPIInt rank;
Mat Gmat = *a_Gmat, tGmat, matTrans;
MPI_Comm comm;
const PetscScalar *vals;
const PetscInt *idx;
PetscInt *d_nnz, *o_nnz;
Vec diag;
MatType mtype;
PetscFunctionBegin;
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventBegin(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
/* scale Gmat for all values between -1 and 1 */
ierr = MatCreateVecs(Gmat, &diag, 0);CHKERRQ(ierr);
ierr = MatGetDiagonal(Gmat, diag);CHKERRQ(ierr);
ierr = VecReciprocal(diag);CHKERRQ(ierr);
ierr = VecSqrtAbs(diag);CHKERRQ(ierr);
ierr = MatDiagonalScale(Gmat, diag, diag);CHKERRQ(ierr);
ierr = VecDestroy(&diag);CHKERRQ(ierr);
if (vfilter < 0.0 && !symm) {
/* Just use the provided matrix as the graph but make all values positive */
MatInfo info;
PetscScalar *avals;
PetscBool isaij,ismpiaij;
ierr = PetscObjectBaseTypeCompare((PetscObject)Gmat,MATSEQAIJ,&isaij);CHKERRQ(ierr);
ierr = PetscObjectBaseTypeCompare((PetscObject)Gmat,MATMPIAIJ,&ismpiaij);CHKERRQ(ierr);
if (!isaij && !ismpiaij) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_USER,"Require (MPI)AIJ matrix type");
if (isaij) {
ierr = MatGetInfo(Gmat,MAT_LOCAL,&info);CHKERRQ(ierr);
ierr = MatSeqAIJGetArray(Gmat,&avals);CHKERRQ(ierr);
for (jj = 0; jj<info.nz_used; jj++) avals[jj] = PetscAbsScalar(avals[jj]);
ierr = MatSeqAIJRestoreArray(Gmat,&avals);CHKERRQ(ierr);
} else {
Mat_MPIAIJ *aij = (Mat_MPIAIJ*)Gmat->data;
ierr = MatGetInfo(aij->A,MAT_LOCAL,&info);CHKERRQ(ierr);
ierr = MatSeqAIJGetArray(aij->A,&avals);CHKERRQ(ierr);
for (jj = 0; jj<info.nz_used; jj++) avals[jj] = PetscAbsScalar(avals[jj]);
ierr = MatSeqAIJRestoreArray(aij->A,&avals);CHKERRQ(ierr);
ierr = MatGetInfo(aij->B,MAT_LOCAL,&info);CHKERRQ(ierr);
ierr = MatSeqAIJGetArray(aij->B,&avals);CHKERRQ(ierr);
for (jj = 0; jj<info.nz_used; jj++) avals[jj] = PetscAbsScalar(avals[jj]);
ierr = MatSeqAIJRestoreArray(aij->B,&avals);CHKERRQ(ierr);
}
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
PetscFunctionReturn(0);
}
ierr = PetscObjectGetComm((PetscObject)Gmat,&comm);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Gmat, &Istart, &Iend);CHKERRQ(ierr);
nloc = Iend - Istart;
ierr = MatGetSize(Gmat, &MM, &NN);CHKERRQ(ierr);
if (symm) {
ierr = MatTranspose(Gmat, MAT_INITIAL_MATRIX, &matTrans);CHKERRQ(ierr);
}
/* Determine upper bound on nonzeros needed in new filtered matrix */
ierr = PetscMalloc2(nloc, &d_nnz,nloc, &o_nnz);CHKERRQ(ierr);
for (Ii = Istart, jj = 0; Ii < Iend; Ii++, jj++) {
ierr = MatGetRow(Gmat,Ii,&ncols,NULL,NULL);CHKERRQ(ierr);
d_nnz[jj] = ncols;
o_nnz[jj] = ncols;
ierr = MatRestoreRow(Gmat,Ii,&ncols,NULL,NULL);CHKERRQ(ierr);
if (symm) {
ierr = MatGetRow(matTrans,Ii,&ncols,NULL,NULL);CHKERRQ(ierr);
d_nnz[jj] += ncols;
o_nnz[jj] += ncols;
ierr = MatRestoreRow(matTrans,Ii,&ncols,NULL,NULL);CHKERRQ(ierr);
}
if (d_nnz[jj] > nloc) d_nnz[jj] = nloc;
if (o_nnz[jj] > (MM-nloc)) o_nnz[jj] = MM - nloc;
}
ierr = MatGetType(Gmat,&mtype);CHKERRQ(ierr);
ierr = MatCreate(comm, &tGmat);CHKERRQ(ierr);
ierr = MatSetSizes(tGmat,nloc,nloc,MM,MM);CHKERRQ(ierr);
ierr = MatSetBlockSizes(tGmat, 1, 1);CHKERRQ(ierr);
ierr = MatSetType(tGmat, mtype);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(tGmat,0,d_nnz);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(tGmat,0,d_nnz,0,o_nnz);CHKERRQ(ierr);
ierr = PetscFree2(d_nnz,o_nnz);CHKERRQ(ierr);
if (symm) {
ierr = MatDestroy(&matTrans);CHKERRQ(ierr);
} else {
/* all entries are generated locally so MatAssembly will be slightly faster for large process counts */
ierr = MatSetOption(tGmat,MAT_NO_OFF_PROC_ENTRIES,PETSC_TRUE);CHKERRQ(ierr);
}
for (Ii = Istart, nnz0 = nnz1 = 0; Ii < Iend; Ii++) {
ierr = MatGetRow(Gmat,Ii,&ncols,&idx,&vals);CHKERRQ(ierr);
for (jj=0; jj<ncols; jj++,nnz0++) {
PetscScalar sv = PetscAbs(PetscRealPart(vals[jj]));
if (PetscRealPart(sv) > vfilter) {
nnz1++;
if (symm) {
sv *= 0.5;
ierr = MatSetValues(tGmat,1,&Ii,1,&idx[jj],&sv,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(tGmat,1,&idx[jj],1,&Ii,&sv,ADD_VALUES);CHKERRQ(ierr);
} else {
ierr = MatSetValues(tGmat,1,&Ii,1,&idx[jj],&sv,ADD_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatRestoreRow(Gmat,Ii,&ncols,&idx,&vals);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(tGmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(tGmat,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);CHKERRQ(ierr);
#endif
#if defined(PETSC_USE_INFO)
{
double t1 = (!nnz0) ? 1. : 100.*(double)nnz1/(double)nnz0, t2 = (!nloc) ? 1. : (double)nnz0/(double)nloc;
ierr = PetscInfo4(*a_Gmat,"\t %g%% nnz after filtering, with threshold %g, %g nnz ave. (N=%D)\n",t1,vfilter,t2,MM);CHKERRQ(ierr);
}
#endif
ierr = MatDestroy(&Gmat);CHKERRQ(ierr);
*a_Gmat = tGmat;
PetscFunctionReturn(0);
}
void _Stokes_SLE_PenaltySolver_Solve( void* solver,void* stokesSLE ) {
Stokes_SLE_PenaltySolver* self = (Stokes_SLE_PenaltySolver*)solver;
Stokes_SLE* sle = (Stokes_SLE*)stokesSLE;
/* Create shortcuts to stuff needed on sle */
Mat kMatrix = sle->kStiffMat->matrix;
Mat gradMat = sle->gStiffMat->matrix;
Mat divMat = NULL;
Mat C_Mat = sle->cStiffMat->matrix;
Vec uVec = sle->uSolnVec->vector;
Vec pVec = sle->pSolnVec->vector;
Vec fVec = sle->fForceVec->vector;
Vec hVec = sle->hForceVec->vector;
Vec hTempVec;
Vec fTempVec;
Vec penalty;
Mat GTrans, kHat;
KSP ksp_v;
double negOne=-1.0;
double one=1.0;
Mat C_InvMat;
Vec diagC;
PC pc;
int rank;
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
Journal_DPrintf( self->debug, "In %s():\n", __func__ );
VecDuplicate( hVec, &hTempVec );
VecDuplicate( fVec, &fTempVec );
VecDuplicate( pVec, &diagC );
if( sle->dStiffMat == NULL ) {
Journal_DPrintf( self->debug, "Div matrix == NULL : Problem is assumed to be symmetric. ie Div = GTrans \n");
#if( PETSC_VERSION_MAJOR <= 2 )
MatTranspose( gradMat, >rans );
#else
MatTranspose( gradMat, MAT_INITIAL_MATRIX, >rans );
#endif
divMat = GTrans;
}
else {
MatType type;
PetscInt size[2];
MatGetType( sle->dStiffMat->matrix, &type );
MatGetLocalSize( sle->dStiffMat->matrix, size + 0, size + 1 );
/* make a copy we can play with */
MatCreate( sle->comm, >rans );
MatSetSizes( GTrans, size[0], size[1], PETSC_DECIDE, PETSC_DECIDE );
MatSetType( GTrans, type );
#if (((PETSC_VERSION_MAJOR==3) && (PETSC_VERSION_MINOR>=3)) || (PETSC_VERSION_MAJOR>3) )
MatSetUp(GTrans);
#endif
MatCopy( sle->dStiffMat->matrix, GTrans, DIFFERENT_NONZERO_PATTERN );
divMat = GTrans;
}
Stokes_SLE_PenaltySolver_MakePenalty( self, sle, &penalty );
/* Create CInv */
MatGetDiagonal( C_Mat, diagC );
VecReciprocal( diagC );
VecPointwiseMult( diagC, penalty, diagC );
{ /* Print the maximum and minimum penalties in my system. */
PetscInt idx;
PetscReal min, max;
VecMin( diagC, &idx, &min );
VecMax( diagC, &idx, &max );
if( rank == 0 ) {
printf( "PENALTY RANGE:\n" );
printf( " MIN: %e\n", min );
printf( " MAX: %e\n", max );
}
}
MatDiagonalSet( C_Mat, diagC, INSERT_VALUES );
C_InvMat = C_Mat; /* Use pointer CInv since C has been inverted */
/* Build RHS : rhs = f - GCInv h */
MatMult( C_InvMat, hVec, hTempVec ); /* hTempVec = C_InvMat * hVec */
VecScale( hTempVec, -1.0 );
MatMult( gradMat, hTempVec, fTempVec );
#if 0
VecPointwiseMult( fTempVec, penalty, fTempVec );
{ /* Print the maximum and minimum penalties in my system. */
PetscInt idx;
PetscReal min, max;
VecMin( fTempVec, &idx, &min );
VecMax( fTempVec, &idx, &max );
printf( "PENALTY RANGE:\n" );
printf( " MIN: %e\n", min );
printf( " MAX: %e\n", max );
}
#endif
VecAXPY( fTempVec, 1.0, fVec );
/*MatMultAdd( gradMat, hTempVec, fVec, fTempVec );*/
/* Build G CInv GTrans */
/* MatTranspose( gradMat, >rans ); */
/* since CInv is diagonal we can just scale mat entries by the diag vector */
MatDiagonalScale( divMat, diagC, PETSC_NULL ); /* Div = CInve Div */
MatMatMult( gradMat, divMat, MAT_INITIAL_MATRIX, PETSC_DEFAULT, &kHat );
/*MatDiagonalScale( kHat, penalty, PETSC_NULL );*/
MatScale( kHat, -1.0 );
MatAXPY( kMatrix, 1.0, kHat, SAME_NONZERO_PATTERN );
/* Setup solver context and make sure that it uses a direct solver */
KSPCreate( sle->comm, &ksp_v );
Stg_KSPSetOperators( ksp_v, kMatrix, kMatrix, DIFFERENT_NONZERO_PATTERN );
KSPSetType( ksp_v, KSPPREONLY );
KSPGetPC( ksp_v, &pc );
PCSetType( pc, PCLU );
KSPSetFromOptions( ksp_v );
KSPSolve( ksp_v, fTempVec, uVec );
/* Recover p */
if( sle->dStiffMat == NULL ) {
/* since Div was modified when C is diagonal, re build the transpose */
if( GTrans != PETSC_NULL )
Stg_MatDestroy(>rans );
#if( PETSC_VERSION_MAJOR <= 2 )
MatTranspose( gradMat, >rans );
#else
MatTranspose( gradMat, MAT_INITIAL_MATRIX, >rans );
#endif
divMat = GTrans;
}
else {
/* never modified Div_null so set divMat to point back to it */
divMat = sle->dStiffMat->matrix;
}
MatMult( divMat, uVec, hTempVec ); /* hTemp = Div v */
VecAYPX( hTempVec, negOne, hVec ); /* hTemp = H - hTemp : hTemp = H - Div v */
MatMult( C_InvMat, hTempVec, pVec ); /* p = CInv hTemp : p = CInv ( H - Div v ) */
Stg_MatDestroy(&kHat );
if( fTempVec != PETSC_NULL ) Stg_VecDestroy(&fTempVec );
if( hTempVec != PETSC_NULL ) Stg_VecDestroy(&hTempVec );
if( diagC != PETSC_NULL ) Stg_VecDestroy(&diagC );
if( ksp_v != PETSC_NULL ) Stg_KSPDestroy(&ksp_v );
if( GTrans != PETSC_NULL ) Stg_MatDestroy(>rans );
}
PetscErrorCode DMPlexPreallocateOperator(DM dm, PetscInt bs, PetscSection section, PetscSection sectionGlobal, PetscInt dnz[], PetscInt onz[], PetscInt dnzu[], PetscInt onzu[], Mat A, PetscBool fillMatrix)
{
MPI_Comm comm;
MatType mtype;
PetscSF sf, sfDof, sfAdj;
PetscSection leafSectionAdj, rootSectionAdj, sectionAdj, anchorSectionAdj;
PetscInt nroots, nleaves, l, p;
const PetscInt *leaves;
const PetscSFNode *remotes;
PetscInt dim, pStart, pEnd, numDof, globalOffStart, globalOffEnd, numCols;
PetscInt *tmpAdj = NULL, *adj, *rootAdj, *anchorAdj = NULL, *cols, *remoteOffsets;
PetscInt adjSize;
PetscLayout rLayout;
PetscInt locRows, rStart, rEnd, r;
PetscMPIInt size;
PetscBool doCommLocal, doComm, debug = PETSC_FALSE, isSymBlock, isSymSeqBlock, isSymMPIBlock;
PetscBool useAnchors;
PetscErrorCode ierr;
PetscFunctionBegin;
PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 3);
PetscValidHeaderSpecific(sectionGlobal, PETSC_SECTION_CLASSID, 4);
PetscValidHeaderSpecific(A, MAT_CLASSID, 9);
if (dnz) PetscValidPointer(dnz,5);
if (onz) PetscValidPointer(onz,6);
if (dnzu) PetscValidPointer(dnzu,7);
if (onzu) PetscValidPointer(onzu,8);
ierr = PetscLogEventBegin(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
ierr = PetscOptionsGetBool(NULL, "-dm_view_preallocation", &debug, NULL);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
ierr = PetscSFGetGraph(sf, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
doCommLocal = (size > 1) && (nroots >= 0) ? PETSC_TRUE : PETSC_FALSE;
ierr = MPI_Allreduce(&doCommLocal, &doComm, 1, MPIU_BOOL, MPI_LAND, comm);CHKERRQ(ierr);
/* Create dof SF based on point SF */
if (debug) {
ierr = PetscPrintf(comm, "Input Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(section, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input Global Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionGlobal, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Input SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sf, NULL);CHKERRQ(ierr);
}
ierr = PetscSFCreateRemoteOffsets(sf, section, section, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sf, section, remoteOffsets, section, &sfDof);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Dof SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfDof, NULL);CHKERRQ(ierr);
}
/* Create section for dof adjacency (dof ==> # adj dof) */
ierr = PetscSectionGetChart(section, &pStart, &pEnd);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(section, &numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(leafSectionAdj, 0, numDof);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, &rootSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(rootSectionAdj, 0, numDof);CHKERRQ(ierr);
/* Fill in the ghost dofs on the interface */
ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, &remotes);CHKERRQ(ierr);
/* use constraints in finding adjacency in this routine */
ierr = DMPlexGetAdjacencyUseAnchors(dm,&useAnchors);CHKERRQ(ierr);
ierr = DMPlexSetAdjacencyUseAnchors(dm,PETSC_TRUE);CHKERRQ(ierr);
/*
section - maps points to (# dofs, local dofs)
sectionGlobal - maps points to (# dofs, global dofs)
leafSectionAdj - maps unowned local dofs to # adj dofs
rootSectionAdj - maps owned local dofs to # adj dofs
adj - adj global dofs indexed by leafSectionAdj
rootAdj - adj global dofs indexed by rootSectionAdj
sf - describes shared points across procs
sfDof - describes shared dofs across procs
sfAdj - describes shared adjacent dofs across procs
** The bootstrapping process involves six rounds with similar structure of visiting neighbors of each point.
(0). If there are point-to-point constraints, add the adjacencies of constrained points to anchors in anchorAdj
(This is done in DMPlexComputeAnchorAdjacencies())
1. Visit unowned points on interface, count adjacencies placing in leafSectionAdj
Reduce those counts to rootSectionAdj (now redundantly counting some interface points)
2. Visit owned points on interface, count adjacencies placing in rootSectionAdj
Create sfAdj connecting rootSectionAdj and leafSectionAdj
3. Visit unowned points on interface, write adjacencies to adj
Gather adj to rootAdj (note that there is redundancy in rootAdj when multiple procs find the same adjacencies)
4. Visit owned points on interface, write adjacencies to rootAdj
Remove redundancy in rootAdj
** The last two traversals use transitive closure
5. Visit all owned points in the subdomain, count dofs for each point (sectionAdj)
Allocate memory addressed by sectionAdj (cols)
6. Visit all owned points in the subdomain, insert dof adjacencies into cols
** Knowing all the column adjacencies, check ownership and sum into dnz and onz
*/
ierr = DMPlexComputeAnchorAdjacencies(dm,section,sectionGlobal,&anchorSectionAdj,&anchorAdj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(leafSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation on Leaves:\n");CHKERRQ(ierr);
ierr = PetscSectionView(leafSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get maximum remote adjacency sizes for owned dofs on interface (roots) */
if (doComm) {
ierr = PetscSFReduceBegin(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
ierr = PetscSFReduceEnd(sfDof, MPIU_INT, leafSectionAdj->atlasDof, rootSectionAdj->atlasDof, MPI_SUM);CHKERRQ(ierr);
}
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Add in local adjacency sizes for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = off; d < off+dof; ++d) {
ierr = PetscSectionAddDof(rootSectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after local additions:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Create adj SF based on dof SF */
ierr = PetscSFCreateRemoteOffsets(sfDof, rootSectionAdj, leafSectionAdj, &remoteOffsets);CHKERRQ(ierr);
ierr = PetscSFCreateSectionSF(sfDof, rootSectionAdj, remoteOffsets, leafSectionAdj, &sfAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency SF for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSFView(sfAdj, NULL);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfDof);CHKERRQ(ierr);
/* Create leaf adjacency */
ierr = PetscSectionSetUp(leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionGetStorageSize(leafSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscCalloc1(adjSize, &adj);CHKERRQ(ierr);
for (l = 0; l < nleaves; ++l) {
PetscInt dof, off, d, q, anDof, anOff;
PetscInt p = leaves[l], numAdj = PETSC_DETERMINE;
if ((p < pStart) || (p >= pEnd)) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt aoff, i = 0;
ierr = PetscSectionGetOffset(leafSectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
adj[aoff+i] = (ngoff < 0 ? -(ngoff+1) : ngoff) + nd;
++i;
}
}
for (q = 0; q < anDof; q++) {
adj[aoff+i] = anchorAdj[anOff+q];
++i;
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Leaf adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, adj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Gather adjacenct indices to root */
ierr = PetscSectionGetStorageSize(rootSectionAdj, &adjSize);CHKERRQ(ierr);
ierr = PetscMalloc1(adjSize, &rootAdj);CHKERRQ(ierr);
for (r = 0; r < adjSize; ++r) rootAdj[r] = -1;
if (doComm) {
ierr = PetscSFGatherBegin(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
ierr = PetscSFGatherEnd(sfAdj, MPIU_INT, adj, rootAdj);CHKERRQ(ierr);
}
ierr = PetscSFDestroy(&sfAdj);CHKERRQ(ierr);
ierr = PetscFree(adj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices after gather\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Add in local adjacency indices for owned dofs on interface (roots) */
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, adof, dof, off, d, q, anDof, anOff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = off; d < off+dof; ++d) {
PetscInt adof, aoff, i;
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
i = adof-1;
for (q = 0; q < anDof; q++) {
rootAdj[aoff+i] = anchorAdj[anOff+q];
--i;
}
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd) {
rootAdj[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
--i;
}
}
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Root adjacency indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Compress indices */
ierr = PetscSectionSetUp(rootSectionAdj);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt dof, cdof, off, d;
PetscInt adof, aoff;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
if (!dof) continue;
ierr = PetscSectionGetDof(rootSectionAdj, off, &adof);CHKERRQ(ierr);
if (adof <= 0) continue;
for (d = off; d < off+dof-cdof; ++d) {
ierr = PetscSectionGetDof(rootSectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, d, &aoff);CHKERRQ(ierr);
ierr = PetscSortRemoveDupsInt(&adof, &rootAdj[aoff]);CHKERRQ(ierr);
ierr = PetscSectionSetDof(rootSectionAdj, d, adof);CHKERRQ(ierr);
}
}
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Adjancency Section for Preallocation on Roots after compression:\n");CHKERRQ(ierr);
ierr = PetscSectionView(rootSectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(comm, "Root adjacency indices after compression\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, adjSize, rootAdj, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Build adjacency section: Maps global indices to sets of adjacent global indices */
ierr = PetscSectionGetOffsetRange(sectionGlobal, &globalOffStart, &globalOffEnd);CHKERRQ(ierr);
ierr = PetscSectionCreate(comm, §ionAdj);CHKERRQ(ierr);
ierr = PetscSectionSetChart(sectionAdj, globalOffStart, globalOffEnd);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
ierr = PetscSectionSetDof(sectionAdj, goff+d, rdof);CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, noff;
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, padj, &noff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, ndof-ncdof);CHKERRQ(ierr);
}
}
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
if (anDof) {
for (d = goff; d < goff+dof-cdof; ++d) {
ierr = PetscSectionAddDof(sectionAdj, d, anDof);CHKERRQ(ierr);
}
}
}
ierr = PetscSectionSetUp(sectionAdj);CHKERRQ(ierr);
if (debug) {
ierr = PetscPrintf(comm, "Adjacency Section for Preallocation:\n");CHKERRQ(ierr);
ierr = PetscSectionView(sectionAdj, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* Get adjacent indices */
ierr = PetscSectionGetStorageSize(sectionAdj, &numCols);CHKERRQ(ierr);
ierr = PetscMalloc1(numCols, &cols);CHKERRQ(ierr);
for (p = pStart; p < pEnd; ++p) {
PetscInt numAdj = PETSC_DETERMINE, dof, cdof, off, goff, d, q, anDof, anOff;
PetscBool found = PETSC_TRUE;
ierr = PetscSectionGetDof(section, p, &dof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, p, &cdof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(section, p, &off);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, p, &goff);CHKERRQ(ierr);
for (d = 0; d < dof-cdof; ++d) {
PetscInt ldof, rdof;
ierr = PetscSectionGetDof(leafSectionAdj, off+d, &ldof);CHKERRQ(ierr);
ierr = PetscSectionGetDof(rootSectionAdj, off+d, &rdof);CHKERRQ(ierr);
if (ldof > 0) {
/* We do not own this point */
} else if (rdof > 0) {
PetscInt aoff, roff;
ierr = PetscSectionGetOffset(sectionAdj, goff+d, &aoff);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(rootSectionAdj, off+d, &roff);CHKERRQ(ierr);
ierr = PetscMemcpy(&cols[aoff], &rootAdj[roff], rdof * sizeof(PetscInt));CHKERRQ(ierr);
} else {
found = PETSC_FALSE;
}
}
if (found) continue;
ierr = DMPlexGetAdjacency(dm, p, &numAdj, &tmpAdj);CHKERRQ(ierr);
ierr = PetscSectionGetDof(anchorSectionAdj, p, &anDof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(anchorSectionAdj, p, &anOff);CHKERRQ(ierr);
for (d = goff; d < goff+dof-cdof; ++d) {
PetscInt adof, aoff, i = 0;
ierr = PetscSectionGetDof(sectionAdj, d, &adof);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, d, &aoff);CHKERRQ(ierr);
for (q = 0; q < numAdj; ++q) {
const PetscInt padj = tmpAdj[q];
PetscInt ndof, ncdof, ngoff, nd;
const PetscInt *ncind;
/* Adjacent points may not be in the section chart */
if ((padj < pStart) || (padj >= pEnd)) continue;
ierr = PetscSectionGetDof(section, padj, &ndof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintDof(section, padj, &ncdof);CHKERRQ(ierr);
ierr = PetscSectionGetConstraintIndices(section, padj, &ncind);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionGlobal, padj, &ngoff);CHKERRQ(ierr);
for (nd = 0; nd < ndof-ncdof; ++nd, ++i) {
cols[aoff+i] = ngoff < 0 ? -(ngoff+1)+nd : ngoff+nd;
}
}
for (q = 0; q < anDof; q++, i++) {
cols[aoff+i] = anchorAdj[anOff + q];
}
if (i != adof) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of entries %D != %D for dof %D (point %D)", i, adof, d, p);
}
}
ierr = PetscSectionDestroy(&anchorSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&leafSectionAdj);CHKERRQ(ierr);
ierr = PetscSectionDestroy(&rootSectionAdj);CHKERRQ(ierr);
ierr = PetscFree(anchorAdj);CHKERRQ(ierr);
ierr = PetscFree(rootAdj);CHKERRQ(ierr);
ierr = PetscFree(tmpAdj);CHKERRQ(ierr);
/* Debugging */
if (debug) {
IS tmp;
ierr = PetscPrintf(comm, "Column indices\n");CHKERRQ(ierr);
ierr = ISCreateGeneral(comm, numCols, cols, PETSC_USE_POINTER, &tmp);CHKERRQ(ierr);
ierr = ISView(tmp, NULL);CHKERRQ(ierr);
ierr = ISDestroy(&tmp);CHKERRQ(ierr);
}
/* Create allocation vectors from adjacency graph */
ierr = MatGetLocalSize(A, &locRows, NULL);CHKERRQ(ierr);
ierr = PetscLayoutCreate(PetscObjectComm((PetscObject)A), &rLayout);CHKERRQ(ierr);
ierr = PetscLayoutSetLocalSize(rLayout, locRows);CHKERRQ(ierr);
ierr = PetscLayoutSetBlockSize(rLayout, 1);CHKERRQ(ierr);
ierr = PetscLayoutSetUp(rLayout);CHKERRQ(ierr);
ierr = PetscLayoutGetRange(rLayout, &rStart, &rEnd);CHKERRQ(ierr);
ierr = PetscLayoutDestroy(&rLayout);CHKERRQ(ierr);
/* Only loop over blocks of rows */
if (rStart%bs || rEnd%bs) SETERRQ3(PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Invalid layout [%d, %d) for matrix, must be divisible by block size %d", rStart, rEnd, bs);
for (r = rStart/bs; r < rEnd/bs; ++r) {
const PetscInt row = r*bs;
PetscInt numCols, cStart, c;
ierr = PetscSectionGetDof(sectionAdj, row, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, row, &cStart);CHKERRQ(ierr);
for (c = cStart; c < cStart+numCols; ++c) {
if ((cols[c] >= rStart*bs) && (cols[c] < rEnd*bs)) {
++dnz[r-rStart];
if (cols[c] >= row) ++dnzu[r-rStart];
} else {
++onz[r-rStart];
if (cols[c] >= row) ++onzu[r-rStart];
}
}
}
if (bs > 1) {
for (r = 0; r < locRows/bs; ++r) {
dnz[r] /= bs;
onz[r] /= bs;
dnzu[r] /= bs;
onzu[r] /= bs;
}
}
/* Set matrix pattern */
ierr = MatXAIJSetPreallocation(A, bs, dnz, onz, dnzu, onzu);CHKERRQ(ierr);
ierr = MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE);CHKERRQ(ierr);
/* Check for symmetric storage */
ierr = MatGetType(A, &mtype);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSBAIJ, &isSymBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock);CHKERRQ(ierr);
ierr = PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock);CHKERRQ(ierr);
if (isSymBlock || isSymSeqBlock || isSymMPIBlock) {ierr = MatSetOption(A, MAT_IGNORE_LOWER_TRIANGULAR, PETSC_TRUE);CHKERRQ(ierr);}
/* Fill matrix with zeros */
if (fillMatrix) {
PetscScalar *values;
PetscInt maxRowLen = 0;
for (r = rStart; r < rEnd; ++r) {
PetscInt len;
ierr = PetscSectionGetDof(sectionAdj, r, &len);CHKERRQ(ierr);
maxRowLen = PetscMax(maxRowLen, len);
}
ierr = PetscCalloc1(maxRowLen, &values);CHKERRQ(ierr);
for (r = rStart; r < rEnd; ++r) {
PetscInt numCols, cStart;
ierr = PetscSectionGetDof(sectionAdj, r, &numCols);CHKERRQ(ierr);
ierr = PetscSectionGetOffset(sectionAdj, r, &cStart);CHKERRQ(ierr);
ierr = MatSetValues(A, 1, &r, numCols, &cols[cStart], values, INSERT_VALUES);CHKERRQ(ierr);
}
ierr = PetscFree(values);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* restore original useAnchors */
ierr = DMPlexSetAdjacencyUseAnchors(dm,useAnchors);CHKERRQ(ierr);
ierr = PetscSectionDestroy(§ionAdj);CHKERRQ(ierr);
ierr = PetscFree(cols);CHKERRQ(ierr);
ierr = PetscLogEventEnd(DMPLEX_Preallocate,dm,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PCGAMGProlongator_Classical_Direct(PC pc, const Mat A, const Mat G, PetscCoarsenData *agg_lists,Mat *P)
{
PetscErrorCode ierr;
MPI_Comm comm;
PetscReal *Amax_pos,*Amax_neg;
Mat lA,gA; /* on and off diagonal matrices */
PetscInt fn; /* fine local blocked sizes */
PetscInt cn; /* coarse local blocked sizes */
PetscInt gn; /* size of the off-diagonal fine vector */
PetscInt fs,fe; /* fine (row) ownership range*/
PetscInt cs,ce; /* coarse (column) ownership range */
PetscInt i,j; /* indices! */
PetscBool iscoarse; /* flag for determining if a node is coarse */
PetscInt *lcid,*gcid; /* on and off-processor coarse unknown IDs */
PetscInt *lsparse,*gsparse; /* on and off-processor sparsity patterns for prolongator */
PetscScalar pij;
const PetscScalar *rval;
const PetscInt *rcol;
PetscScalar g_pos,g_neg,a_pos,a_neg,diag,invdiag,alpha,beta;
Vec F; /* vec of coarse size */
Vec C; /* vec of fine size */
Vec gF; /* vec of off-diagonal fine size */
MatType mtype;
PetscInt c_indx;
PetscScalar c_scalar;
PetscInt ncols,col;
PetscInt row_f,row_c;
PetscInt cmax=0,idx;
PetscScalar *pvals;
PetscInt *pcols;
PC_MG *mg = (PC_MG*)pc->data;
PC_GAMG *gamg = (PC_GAMG*)mg->innerctx;
PetscFunctionBegin;
comm = ((PetscObject)pc)->comm;
ierr = MatGetOwnershipRange(A,&fs,&fe); CHKERRQ(ierr);
fn = (fe - fs);
ierr = MatGetVecs(A,&F,NULL);CHKERRQ(ierr);
/* get the number of local unknowns and the indices of the local unknowns */
ierr = PetscMalloc(sizeof(PetscInt)*fn,&lsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&gsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&lcid);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscReal)*fn,&Amax_pos);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscReal)*fn,&Amax_neg);CHKERRQ(ierr);
/* count the number of coarse unknowns */
cn = 0;
for (i=0;i<fn;i++) {
/* filter out singletons */
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
lcid[i] = -1;
if (!iscoarse) {
cn++;
}
}
/* create the coarse vector */
ierr = VecCreateMPI(comm,cn,PETSC_DECIDE,&C);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(C,&cs,&ce);CHKERRQ(ierr);
/* construct a global vector indicating the global indices of the coarse unknowns */
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
if (!iscoarse) {
lcid[i] = cs+cn;
cn++;
} else {
lcid[i] = -1;
}
*((PetscInt *)&c_scalar) = lcid[i];
c_indx = fs+i;
ierr = VecSetValues(F,1,&c_indx,&c_scalar,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecAssemblyBegin(F);CHKERRQ(ierr);
ierr = VecAssemblyEnd(F);CHKERRQ(ierr);
/* determine the biggest off-diagonal entries in each row */
for (i=fs;i<fe;i++) {
Amax_pos[i-fs] = 0.;
Amax_neg[i-fs] = 0.;
ierr = MatGetRow(A,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
for(j=0;j<ncols;j++){
if ((PetscRealPart(-rval[j]) > Amax_neg[i-fs]) && i != rcol[j]) Amax_neg[i-fs] = PetscAbsScalar(rval[j]);
if ((PetscRealPart(rval[j]) > Amax_pos[i-fs]) && i != rcol[j]) Amax_pos[i-fs] = PetscAbsScalar(rval[j]);
}
if (ncols > cmax) cmax = ncols;
ierr = MatRestoreRow(A,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
}
ierr = PetscMalloc(sizeof(PetscInt)*cmax,&pcols);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*cmax,&pvals);CHKERRQ(ierr);
/* split the operator into two */
ierr = PCGAMGClassicalGraphSplitting_Private(A,&lA,&gA);CHKERRQ(ierr);
/* scatter to the ghost vector */
ierr = PCGAMGClassicalCreateGhostVector_Private(A,&gF,NULL);CHKERRQ(ierr);
ierr = PCGAMGClassicalGhost_Private(A,F,gF);CHKERRQ(ierr);
if (gA) {
ierr = VecGetSize(gF,&gn);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*gn,&gcid);CHKERRQ(ierr);
for (i=0;i<gn;i++) {
ierr = VecGetValues(gF,1,&i,&c_scalar);CHKERRQ(ierr);
gcid[i] = *((PetscInt *)&c_scalar);
}
}
ierr = VecDestroy(&F);CHKERRQ(ierr);
ierr = VecDestroy(&gF);CHKERRQ(ierr);
ierr = VecDestroy(&C);CHKERRQ(ierr);
/* count the on and off processor sparsity patterns for the prolongator */
for (i=0;i<fn;i++) {
/* on */
lsparse[i] = 0;
gsparse[i] = 0;
if (lcid[i] >= 0) {
lsparse[i] = 1;
gsparse[i] = 0;
} else {
ierr = MatGetRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
for (j = 0;j < ncols;j++) {
col = rcol[j];
if (lcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
lsparse[i] += 1;
}
}
ierr = MatRestoreRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
/* off */
if (gA) {
ierr = MatGetRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
for (j = 0; j < ncols; j++) {
col = rcol[j];
if (gcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
gsparse[i] += 1;
}
}
ierr = MatRestoreRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
}
}
}
/* preallocate and create the prolongator */
ierr = MatCreate(comm,P); CHKERRQ(ierr);
ierr = MatGetType(G,&mtype);CHKERRQ(ierr);
ierr = MatSetType(*P,mtype);CHKERRQ(ierr);
ierr = MatSetSizes(*P,fn,cn,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*P,0,lsparse,0,gsparse);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*P,0,lsparse);CHKERRQ(ierr);
/* loop over local fine nodes -- get the diagonal, the sum of positive and negative strong and weak weights, and set up the row */
for (i = 0;i < fn;i++) {
/* determine on or off */
row_f = i + fs;
row_c = lcid[i];
if (row_c >= 0) {
pij = 1.;
ierr = MatSetValues(*P,1,&row_f,1,&row_c,&pij,INSERT_VALUES);CHKERRQ(ierr);
} else {
g_pos = 0.;
g_neg = 0.;
a_pos = 0.;
a_neg = 0.;
diag = 0.;
/* local connections */
ierr = MatGetRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
for (j = 0; j < ncols; j++) {
col = rcol[j];
if (lcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
if (PetscRealPart(rval[j]) > 0.) {
g_pos += rval[j];
} else {
g_neg += rval[j];
}
}
if (col != i) {
if (PetscRealPart(rval[j]) > 0.) {
a_pos += rval[j];
} else {
a_neg += rval[j];
}
} else {
diag = rval[j];
}
}
ierr = MatRestoreRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
/* ghosted connections */
if (gA) {
ierr = MatGetRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
for (j = 0; j < ncols; j++) {
col = rcol[j];
if (gcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
if (PetscRealPart(rval[j]) > 0.) {
g_pos += rval[j];
} else {
g_neg += rval[j];
}
}
if (PetscRealPart(rval[j]) > 0.) {
a_pos += rval[j];
} else {
a_neg += rval[j];
}
}
ierr = MatRestoreRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
}
if (g_neg == 0.) {
alpha = 0.;
} else {
alpha = -a_neg/g_neg;
}
if (g_pos == 0.) {
diag += a_pos;
beta = 0.;
} else {
beta = -a_pos/g_pos;
}
if (diag == 0.) {
invdiag = 0.;
} else invdiag = 1. / diag;
/* on */
ierr = MatGetRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
idx = 0;
for (j = 0;j < ncols;j++) {
col = rcol[j];
if (lcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
row_f = i + fs;
row_c = lcid[col];
/* set the values for on-processor ones */
if (PetscRealPart(rval[j]) < 0.) {
pij = rval[j]*alpha*invdiag;
} else {
pij = rval[j]*beta*invdiag;
}
if (PetscAbsScalar(pij) != 0.) {
pvals[idx] = pij;
pcols[idx] = row_c;
idx++;
}
}
}
ierr = MatRestoreRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
/* off */
if (gA) {
ierr = MatGetRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
for (j = 0; j < ncols; j++) {
col = rcol[j];
if (gcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
row_f = i + fs;
row_c = gcid[col];
/* set the values for on-processor ones */
if (PetscRealPart(rval[j]) < 0.) {
pij = rval[j]*alpha*invdiag;
} else {
pij = rval[j]*beta*invdiag;
}
if (PetscAbsScalar(pij) != 0.) {
pvals[idx] = pij;
pcols[idx] = row_c;
idx++;
}
}
}
ierr = MatRestoreRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
}
ierr = MatSetValues(*P,1,&row_f,idx,pcols,pvals,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscFree(lsparse);CHKERRQ(ierr);
ierr = PetscFree(gsparse);CHKERRQ(ierr);
ierr = PetscFree(pcols);CHKERRQ(ierr);
ierr = PetscFree(pvals);CHKERRQ(ierr);
ierr = PetscFree(Amax_pos);CHKERRQ(ierr);
ierr = PetscFree(Amax_neg);CHKERRQ(ierr);
ierr = PetscFree(lcid);CHKERRQ(ierr);
if (gA) {ierr = PetscFree(gcid);CHKERRQ(ierr);}
PetscFunctionReturn(0);
}
PetscErrorCode PCGAMGProlongator_Classical_Standard(PC pc, const Mat A, const Mat G, PetscCoarsenData *agg_lists,Mat *P)
{
PetscErrorCode ierr;
Mat *lA;
Vec lv,v,cv;
PetscScalar *lcid;
IS lis;
PetscInt fs,fe,cs,ce,nl,i,j,k,li,lni,ci;
VecScatter lscat;
PetscInt fn,cn,cid,c_indx;
PetscBool iscoarse;
PetscScalar c_scalar;
const PetscScalar *vcol;
const PetscInt *icol;
const PetscInt *gidx;
PetscInt ncols;
PetscInt *lsparse,*gsparse;
MatType mtype;
PetscInt maxcols;
PetscReal diag,jdiag,jwttotal;
PetscScalar *pvcol,vi;
PetscInt *picol;
PetscInt pncols;
PetscScalar *pcontrib,pentry,pjentry;
/* PC_MG *mg = (PC_MG*)pc->data; */
/* PC_GAMG *gamg = (PC_GAMG*)mg->innerctx; */
PetscFunctionBegin;
ierr = MatGetOwnershipRange(A,&fs,&fe);CHKERRQ(ierr);
fn = fe-fs;
ierr = MatGetVecs(A,NULL,&v);CHKERRQ(ierr);
ierr = ISCreateStride(PETSC_COMM_SELF,fe-fs,fs,1,&lis);CHKERRQ(ierr);
/* increase the overlap by two to get neighbors of neighbors */
ierr = MatIncreaseOverlap(A,1,&lis,2);CHKERRQ(ierr);
ierr = ISSort(lis);CHKERRQ(ierr);
/* get the local part of A */
ierr = MatGetSubMatrices(A,1,&lis,&lis,MAT_INITIAL_MATRIX,&lA);CHKERRQ(ierr);
/* build the scatter out of it */
ierr = ISGetLocalSize(lis,&nl);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,nl,&lv);CHKERRQ(ierr);
ierr = VecScatterCreate(v,lis,lv,NULL,&lscat);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&lsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*fn,&gsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*nl,&pcontrib);CHKERRQ(ierr);
/* create coarse vector */
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse);CHKERRQ(ierr);
if (!iscoarse) {
cn++;
}
}
ierr = VecCreateMPI(PetscObjectComm((PetscObject)A),cn,PETSC_DECIDE,&cv);CHKERRQ(ierr);
ierr = VecGetOwnershipRange(cv,&cs,&ce);CHKERRQ(ierr);
cn = 0;
for (i=0;i<fn;i++) {
ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
if (!iscoarse) {
cid = cs+cn;
cn++;
} else {
cid = -1;
}
*(PetscInt*)&c_scalar = cid;
c_indx = fs+i;
ierr = VecSetValues(v,1,&c_indx,&c_scalar,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = VecScatterBegin(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
ierr = VecScatterEnd(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
/* count to preallocate the prolongator */
ierr = ISGetIndices(lis,&gidx);CHKERRQ(ierr);
ierr = VecGetArray(lv,&lcid);CHKERRQ(ierr);
maxcols = 0;
/* count the number of unique contributing coarse cells for each fine */
for (i=0;i<nl;i++) {
pcontrib[i] = 0.;
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
if (gidx[i] >= fs && gidx[i] < fe) {
li = gidx[i] - fs;
lsparse[li] = 0;
gsparse[li] = 0;
cid = *(PetscInt*)&(lcid[i]);
if (cid >= 0) {
lsparse[li] = 1;
} else {
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0) {
pcontrib[icol[j]] = 1.;
} else {
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0) {
pcontrib[icol[k]] = 1.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
}
}
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0 && pcontrib[icol[j]] != 0.) {
lni = *(PetscInt*)&(lcid[icol[j]]);
if (lni >= cs && lni < ce) {
lsparse[li]++;
} else {
gsparse[li]++;
}
pcontrib[icol[j]] = 0.;
} else {
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && pcontrib[icol[k]] != 0.) {
lni = *(PetscInt*)&(lcid[icol[k]]);
if (lni >= cs && lni < ce) {
lsparse[li]++;
} else {
gsparse[li]++;
}
pcontrib[icol[k]] = 0.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,NULL);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,NULL);CHKERRQ(ierr);
}
}
}
if (lsparse[li] + gsparse[li] > maxcols) maxcols = lsparse[li]+gsparse[li];
}
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
ierr = PetscMalloc(sizeof(PetscInt)*maxcols,&picol);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*maxcols,&pvcol);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)A),P);CHKERRQ(ierr);
ierr = MatGetType(A,&mtype);CHKERRQ(ierr);
ierr = MatSetType(*P,mtype);CHKERRQ(ierr);
ierr = MatSetSizes(*P,fn,cn,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*P,0,lsparse,0,gsparse);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*P,0,lsparse);CHKERRQ(ierr);
for (i=0;i<nl;i++) {
diag = 0.;
if (gidx[i] >= fs && gidx[i] < fe) {
li = gidx[i] - fs;
pncols=0;
cid = *(PetscInt*)&(lcid[i]);
if (cid >= 0) {
pncols = 1;
picol[0] = cid;
pvcol[0] = 1.;
} else {
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
for (j=0;j<ncols;j++) {
pentry = vcol[j];
if (*(PetscInt*)&(lcid[icol[j]]) >= 0) {
/* coarse neighbor */
pcontrib[icol[j]] += pentry;
} else if (icol[j] != i) {
/* the neighbor is a strongly connected fine node */
ci = icol[j];
vi = vcol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
jwttotal=0.;
jdiag = 0.;
for (k=0;k<ncols;k++) {
if (ci == icol[k]) {
jdiag = PetscRealPart(vcol[k]);
}
}
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && jdiag*PetscRealPart(vcol[k]) < 0.) {
pjentry = vcol[k];
jwttotal += PetscRealPart(pjentry);
}
}
if (jwttotal != 0.) {
jwttotal = PetscRealPart(vi)/jwttotal;
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && jdiag*PetscRealPart(vcol[k]) < 0.) {
pjentry = vcol[k]*jwttotal;
pcontrib[icol[k]] += pjentry;
}
}
} else {
diag += PetscRealPart(vi);
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
} else {
diag += PetscRealPart(vcol[j]);
}
}
if (diag != 0.) {
diag = 1./diag;
for (j=0;j<ncols;j++) {
if (*(PetscInt*)&(lcid[icol[j]]) >= 0 && pcontrib[icol[j]] != 0.) {
/* the neighbor is a coarse node */
if (PetscAbsScalar(pcontrib[icol[j]]) > 0.0) {
lni = *(PetscInt*)&(lcid[icol[j]]);
pvcol[pncols] = -pcontrib[icol[j]]*diag;
picol[pncols] = lni;
pncols++;
}
pcontrib[icol[j]] = 0.;
} else {
/* the neighbor is a strongly connected fine node */
ci = icol[j];
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
for (k=0;k<ncols;k++) {
if (*(PetscInt*)&(lcid[icol[k]]) >= 0 && pcontrib[icol[k]] != 0.) {
if (PetscAbsScalar(pcontrib[icol[k]]) > 0.0) {
lni = *(PetscInt*)&(lcid[icol[k]]);
pvcol[pncols] = -pcontrib[icol[k]]*diag;
picol[pncols] = lni;
pncols++;
}
pcontrib[icol[k]] = 0.;
}
}
ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
pcontrib[icol[j]] = 0.;
}
ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
}
}
ci = gidx[i];
li = gidx[i] - fs;
if (pncols > 0) {
ierr = MatSetValues(*P,1,&ci,pncols,picol,pvcol,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = ISRestoreIndices(lis,&gidx);CHKERRQ(ierr);
ierr = VecRestoreArray(lv,&lcid);CHKERRQ(ierr);
ierr = PetscFree(pcontrib);CHKERRQ(ierr);
ierr = PetscFree(picol);CHKERRQ(ierr);
ierr = PetscFree(pvcol);CHKERRQ(ierr);
ierr = PetscFree(lsparse);CHKERRQ(ierr);
ierr = PetscFree(gsparse);CHKERRQ(ierr);
ierr = ISDestroy(&lis);CHKERRQ(ierr);
ierr = MatDestroyMatrices(1,&lA);CHKERRQ(ierr);
ierr = VecDestroy(&lv);CHKERRQ(ierr);
ierr = VecDestroy(&cv);CHKERRQ(ierr);
ierr = VecDestroy(&v);CHKERRQ(ierr);
ierr = VecScatterDestroy(&lscat);CHKERRQ(ierr);
ierr = MatAssemblyBegin(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/*
Mat Pold;
ierr = PCGAMGProlongator_Classical(pc,A,G,agg_lists,&Pold);CHKERRQ(ierr);
ierr = MatView(Pold,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatView(*P,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&Pold);CHKERRQ(ierr);
*/
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
PetscMPIInt size;
PetscErrorCode ierr;
Vec x,y,b,s1,s2;
Mat A; /* linear system matrix */
Mat sA,sB,sC; /* symmetric part of the matrices */
PetscInt n,mbs=16,bs=1,nz=3,prob=1,i,j,k1,k2,col[3],lf,block, row,Ii,J,n1,inc;
PetscReal norm1,norm2,rnorm,tol=PETSC_SMALL;
PetscScalar neg_one = -1.0,four=4.0,value[3];
IS perm, iscol;
PetscRandom rdm;
PetscBool doIcc=PETSC_TRUE,equal;
MatInfo minfo1,minfo2;
MatFactorInfo factinfo;
MatType type;
PetscInitialize(&argc,&args,(char*)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
if (size != 1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP,"This is a uniprocessor example only!");
ierr = PetscOptionsGetInt(NULL,"-bs",&bs,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,"-mbs",&mbs,NULL);CHKERRQ(ierr);
n = mbs*bs;
ierr = MatCreate(PETSC_COMM_SELF,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(A,MATSEQBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(A,bs,nz,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_SELF,&sA);CHKERRQ(ierr);
ierr = MatSetSizes(sA,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetType(sA,MATSEQSBAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(sA);CHKERRQ(ierr);
ierr = MatGetType(sA,&type);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)sA,MATSEQSBAIJ,&doIcc);CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(sA,bs,nz,NULL);CHKERRQ(ierr);
ierr = MatSetOption(sA,MAT_IGNORE_LOWER_TRIANGULAR,PETSC_TRUE);CHKERRQ(ierr);
/* Test MatGetOwnershipRange() */
ierr = MatGetOwnershipRange(A,&Ii,&J);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(sA,&i,&j);CHKERRQ(ierr);
if (i-Ii || j-J) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetOwnershipRange() in MatSBAIJ format\n");CHKERRQ(ierr);
}
/* Assemble matrix */
if (bs == 1) {
ierr = PetscOptionsGetInt(NULL,"-test_problem",&prob,NULL);CHKERRQ(ierr);
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0; value[1] = 2.0; value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = n - 1; col[0]=0; col[1] = n - 2; col[2] = n - 1;
value[0]= 0.1; value[1]=-1; value[2]=2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0;
col[0] = n-1; col[1] = 1; col[2] = 0;
value[0] = 0.1; value[1] = -1.0; value[2] = 2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (PetscInt) (PetscSqrtReal((PetscReal)n) + 0.001);
if (n1*n1 - n) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"sqrt(n) must be a positive interger!");
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatSetValues(A,1,&Ii,1,&Ii,&four,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&Ii,1,&Ii,&four,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0; value[1] = 4.0; value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1; col[1] = i; col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
i = bs - 1+block*bs; col[0] = bs - 2+block*bs; col[1] = bs - 1+block*bs;
value[0]=-1.0; value[1]=4.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
i = 0+block*bs; col[0] = 0+block*bs; col[1] = 1+block*bs;
value[0]=4.0; value[1] = -1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(A,1,&i,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&i,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
col[0]=i; row=i+bs;
ierr = MatSetValues(A,1,&row,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatSetValues(sA,1,&row,1,col,value,INSERT_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyBegin(sA,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(sA,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* Test MatGetInfo() of A and sA */
ierr = MatGetInfo(A,MAT_LOCAL,&minfo1);CHKERRQ(ierr);
ierr = MatGetInfo(sA,MAT_LOCAL,&minfo2);CHKERRQ(ierr);
/*
printf("A matrix nonzeros (BAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo1.nz_used,(int)minfo1.nz_allocated);
printf("sA matrix nonzeros(SBAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo2.nz_used,(int)minfo2.nz_allocated);
*/
i = (int) (minfo1.nz_used - minfo2.nz_used);
j = (int) (minfo1.nz_allocated - minfo2.nz_allocated);
k1 = (int) (minfo1.nz_allocated - minfo1.nz_used);
k2 = (int) (minfo2.nz_allocated - minfo2.nz_used);
if (i < 0 || j < 0 || k1 < 0 || k2 < 0) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error (compare A and sA): MatGetInfo()\n");CHKERRQ(ierr);
}
/* Test MatDuplicate() */
ierr = MatNorm(A,NORM_FROBENIUS,&norm1);CHKERRQ(ierr);
ierr = MatDuplicate(sA,MAT_COPY_VALUES,&sB);CHKERRQ(ierr);
ierr = MatEqual(sA,sB,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDuplicate()");
/* Test MatNorm() */
ierr = MatNorm(A,NORM_FROBENIUS,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_FROBENIUS,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_FROBENIUS, NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_INFINITY,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_INFINITY,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_INFINITY(), NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_1,&norm1);CHKERRQ(ierr);
ierr = MatNorm(sB,NORM_1,&norm2);CHKERRQ(ierr);
rnorm = PetscAbsReal(norm1-norm2)/norm2;
if (rnorm > tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_INFINITY(), NormA=%16.14e NormsB=%16.14e\n",norm1,norm2);CHKERRQ(ierr);
}
/* Test MatGetInfo(), MatGetSize(), MatGetBlockSize() */
ierr = MatGetInfo(A,MAT_LOCAL,&minfo1);CHKERRQ(ierr);
ierr = MatGetInfo(sB,MAT_LOCAL,&minfo2);CHKERRQ(ierr);
/*
printf("matrix nonzeros (BAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo1.nz_used,(int)minfo1.nz_allocated);
printf("matrix nonzeros(SBAIJ format) = %d, allocated nonzeros= %d\n", (int)minfo2.nz_used,(int)minfo2.nz_allocated);
*/
i = (int) (minfo1.nz_used - minfo2.nz_used);
j = (int) (minfo1.nz_allocated - minfo2.nz_allocated);
k1 = (int) (minfo1.nz_allocated - minfo1.nz_used);
k2 = (int) (minfo2.nz_allocated - minfo2.nz_used);
if (i < 0 || j < 0 || k1 < 0 || k2 < 0) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error(compare A and sB): MatGetInfo()\n");CHKERRQ(ierr);
}
ierr = MatGetSize(A,&Ii,&J);CHKERRQ(ierr);
ierr = MatGetSize(sB,&i,&j);CHKERRQ(ierr);
if (i-Ii || j-J) {
PetscPrintf(PETSC_COMM_SELF,"Error: MatGetSize()\n");CHKERRQ(ierr);
}
ierr = MatGetBlockSize(A, &Ii);CHKERRQ(ierr);
ierr = MatGetBlockSize(sB, &i);CHKERRQ(ierr);
if (i-Ii) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetBlockSize()\n");CHKERRQ(ierr);
}
ierr = PetscRandomCreate(PETSC_COMM_SELF,&rdm);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rdm);CHKERRQ(ierr);
ierr = VecCreateSeq(PETSC_COMM_SELF,n,&x);CHKERRQ(ierr);
ierr = VecDuplicate(x,&s1);CHKERRQ(ierr);
ierr = VecDuplicate(x,&s2);CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);CHKERRQ(ierr);
ierr = VecDuplicate(x,&b);CHKERRQ(ierr);
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
/* Test MatDiagonalScale(), MatGetDiagonal(), MatScale() */
#if !defined(PETSC_USE_COMPLEX)
/* Scaling matrix with complex numbers results non-spd matrix,
causing crash of MatForwardSolve() and MatBackwardSolve() */
ierr = MatDiagonalScale(A,x,x);CHKERRQ(ierr);
ierr = MatDiagonalScale(sB,x,x);CHKERRQ(ierr);
ierr = MatMultEqual(A,sB,10,&equal);CHKERRQ(ierr);
if (!equal) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDiagonalScale");
ierr = MatGetDiagonal(A,s1);CHKERRQ(ierr);
ierr = MatGetDiagonal(sB,s2);CHKERRQ(ierr);
ierr = VecAXPY(s2,neg_one,s1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm1);CHKERRQ(ierr);
if (norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatGetDiagonal(), ||s1-s2||=%G\n",norm1);CHKERRQ(ierr);
}
{
PetscScalar alpha=0.1;
ierr = MatScale(A,alpha);CHKERRQ(ierr);
ierr = MatScale(sB,alpha);CHKERRQ(ierr);
}
#endif
/* Test MatGetRowMaxAbs() */
ierr = MatGetRowMaxAbs(A,s1,NULL);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(sB,s2,NULL);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatGetRowMaxAbs() \n");CHKERRQ(ierr);
}
/* Test MatMult() */
for (i=0; i<40; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = MatMult(A,x,s1);CHKERRQ(ierr);
ierr = MatMult(sB,x,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatMult(), norm1-norm2: %G\n",norm1);CHKERRQ(ierr);
}
}
/* MatMultAdd() */
for (i=0; i<40; i++) {
ierr = VecSetRandom(x,rdm);CHKERRQ(ierr);
ierr = VecSetRandom(y,rdm);CHKERRQ(ierr);
ierr = MatMultAdd(A,x,y,s1);CHKERRQ(ierr);
ierr = MatMultAdd(sB,x,y,s2);CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&norm1);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&norm2);CHKERRQ(ierr);
norm1 -= norm2;
if (norm1<-tol || norm1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error:MatMultAdd(), norm1-norm2: %G\n",norm1);CHKERRQ(ierr);
}
}
/* Test MatCholeskyFactor(), MatICCFactor() with natural ordering */
ierr = MatGetOrdering(A,MATORDERINGNATURAL,&perm,&iscol);CHKERRQ(ierr);
ierr = ISDestroy(&iscol);CHKERRQ(ierr);
norm1 = tol;
inc = bs;
/* initialize factinfo */
ierr = PetscMemzero(&factinfo,sizeof(MatFactorInfo));CHKERRQ(ierr);
for (lf=-1; lf<10; lf += inc) {
if (lf==-1) { /* Cholesky factor of sB (duplicate sA) */
factinfo.fill = 5.0;
ierr = MatGetFactor(sB,MATSOLVERPETSC,MAT_FACTOR_CHOLESKY,&sC);CHKERRQ(ierr);
ierr = MatCholeskyFactorSymbolic(sC,sB,perm,&factinfo);CHKERRQ(ierr);
} else if (!doIcc) break;
else { /* incomplete Cholesky factor */
factinfo.fill = 5.0;
factinfo.levels = lf;
ierr = MatGetFactor(sB,MATSOLVERPETSC,MAT_FACTOR_ICC,&sC);CHKERRQ(ierr);
ierr = MatICCFactorSymbolic(sC,sB,perm,&factinfo);CHKERRQ(ierr);
}
ierr = MatCholeskyFactorNumeric(sC,sB,&factinfo);CHKERRQ(ierr);
/* MatView(sC, PETSC_VIEWER_DRAW_WORLD); */
/* test MatGetDiagonal on numeric factor */
/*
if (lf == -1) {
ierr = MatGetDiagonal(sC,s1);CHKERRQ(ierr);
printf(" in ex74.c, diag: \n");
ierr = VecView(s1,PETSC_VIEWER_STDOUT_SELF);CHKERRQ(ierr);
}
*/
ierr = MatMult(sB,x,b);CHKERRQ(ierr);
/* test MatForwardSolve() and MatBackwardSolve() */
if (lf == -1) {
ierr = MatForwardSolve(sC,b,s1);CHKERRQ(ierr);
ierr = MatBackwardSolve(sC,s1,s2);CHKERRQ(ierr);
ierr = VecAXPY(s2,neg_one,x);CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_2,&norm2);CHKERRQ(ierr);
if (10*norm1 < norm2) {
ierr = PetscPrintf(PETSC_COMM_SELF,"MatForwardSolve and BackwardSolve: Norm of error=%G, bs=%d\n",norm2,bs);CHKERRQ(ierr);
}
}
/* test MatSolve() */
ierr = MatSolve(sC,b,y);CHKERRQ(ierr);
ierr = MatDestroy(&sC);CHKERRQ(ierr);
/* Check the error */
ierr = VecAXPY(y,neg_one,x);CHKERRQ(ierr);
ierr = VecNorm(y,NORM_2,&norm2);CHKERRQ(ierr);
/* printf("lf: %d, error: %G\n", lf,norm2); */
if (10*norm1 < norm2 && lf-inc != -1) {
ierr = PetscPrintf(PETSC_COMM_SELF,"lf=%D, %D, Norm of error=%G, %G\n",lf-inc,lf,norm1,norm2);CHKERRQ(ierr);
}
norm1 = norm2;
if (norm2 < tol && lf != -1) break;
}
ierr = ISDestroy(&perm);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = MatDestroy(&sB);CHKERRQ(ierr);
ierr = MatDestroy(&sA);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&y);CHKERRQ(ierr);
ierr = VecDestroy(&s1);CHKERRQ(ierr);
ierr = VecDestroy(&s2);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rdm);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
int main(int argc,char **args)
{
Mat C,A;
PetscInt i, n = 10,midx[3],bs=1;
PetscErrorCode ierr;
PetscScalar v[3];
PetscBool flg,isAIJ;
MatType type;
PetscMPIInt size;
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-n",&n,PETSC_NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(PETSC_NULL,"-mat_block_size",&bs,PETSC_NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetType(C,MATAIJ);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatGetType(C,&type);CHKERRQ(ierr);
if (size == 1){
ierr = PetscObjectTypeCompare((PetscObject)C,MATSEQAIJ,&isAIJ);CHKERRQ(ierr);
} else {
ierr = PetscObjectTypeCompare((PetscObject)C,MATMPIAIJ,&isAIJ);CHKERRQ(ierr);
}
ierr = MatSeqAIJSetPreallocation(C,3,PETSC_NULL);
ierr = MatMPIAIJSetPreallocation(C,3,PETSC_NULL,3,PETSC_NULL);CHKERRQ(ierr);
ierr = MatSeqBAIJSetPreallocation(C,bs,3,PETSC_NULL);
ierr = MatMPIBAIJSetPreallocation(C,bs,3,PETSC_NULL,3,PETSC_NULL);CHKERRQ(ierr);
v[0] = -1.; v[1] = 2.; v[2] = -1.;
for (i=1; i<n-1; i++){
midx[2] = i-1; midx[1] = i; midx[0] = i+1;
ierr = MatSetValues(C,1,&i,3,midx,v,INSERT_VALUES);CHKERRQ(ierr);
}
i = 0; midx[0] = 0; midx[1] = 1;
v[0] = 2.0; v[1] = -1.;
ierr = MatSetValues(C,1,&i,2,midx,v,INSERT_VALUES);CHKERRQ(ierr);
i = n-1; midx[0] = n-2; midx[1] = n-1;
v[0] = -1.0; v[1] = 2.;
ierr = MatSetValues(C,1,&i,2,midx,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* test matrices with different nonzero patterns - Note: A is created with different nonzero pattern of C! */
ierr = MatCopy(C,A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatEqual(A,C,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"MatCopy(C,A,DIFFERENT_NONZERO_PATTERN): Matrices are NOT equal");
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"A is obtained with MatCopy(,,DIFFERENT_NONZERO_PATTERN):\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
/* test matrices with same nonzero pattern */
ierr = MatDuplicate(C,MAT_DO_NOT_COPY_VALUES,&A);CHKERRQ(ierr);
ierr = MatCopy(C,A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = MatEqual(A,C,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,1,"MatCopy(C,A,SAME_NONZERO_PATTERN): Matrices are NOT equal");
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nA is obtained with MatCopy(,,SAME_NONZERO_PATTERN):\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerSetFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_COMMON);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"A:\n");CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
/* test MatStore/RetrieveValues() */
if (isAIJ){
ierr = MatSetOption(A,MAT_NEW_NONZERO_LOCATIONS,PETSC_FALSE);CHKERRQ(ierr);
ierr = MatStoreValues(A);CHKERRQ(ierr);
ierr = MatZeroEntries(A);CHKERRQ(ierr);
ierr = MatRetrieveValues(A);CHKERRQ(ierr);
}
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return 0;
}
PetscErrorCode PCGAMGProlongator_GEO(PC pc,Mat Amat,Mat Gmat,PetscCoarsenData *agg_lists,Mat *a_P_out)
{
PC_MG *mg = (PC_MG*)pc->data;
PC_GAMG *pc_gamg = (PC_GAMG*)mg->innerctx;
const PetscInt dim = pc_gamg->data_cell_cols, data_cols = pc_gamg->data_cell_cols;
PetscErrorCode ierr;
PetscInt Istart,Iend,nloc,my0,jj,kk,ncols,nLocalSelected,bs,*clid_flid;
Mat Prol;
PetscMPIInt rank, size;
MPI_Comm comm;
IS selected_2,selected_1;
const PetscInt *selected_idx;
MatType mtype;
PetscFunctionBegin;
ierr = PetscObjectGetComm((PetscObject)Amat,&comm);CHKERRQ(ierr);
ierr = PetscLogEventBegin(PC_GAMGProlongator_GEO,0,0,0,0);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm,&rank);CHKERRQ(ierr);
ierr = MPI_Comm_size(comm,&size);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Amat, &Istart, &Iend);CHKERRQ(ierr);
ierr = MatGetBlockSize(Amat, &bs);CHKERRQ(ierr);
nloc = (Iend-Istart)/bs; my0 = Istart/bs;
if ((Iend-Istart) % bs) SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_PLIB,"(Iend %D - Istart %D) % bs %D",Iend,Istart,bs);
/* get 'nLocalSelected' */
ierr = PetscCDGetMIS(agg_lists, &selected_1);CHKERRQ(ierr);
ierr = ISGetSize(selected_1, &jj);CHKERRQ(ierr);
ierr = PetscMalloc1(jj, &clid_flid);CHKERRQ(ierr);
ierr = ISGetIndices(selected_1, &selected_idx);CHKERRQ(ierr);
for (kk=0,nLocalSelected=0; kk<jj; kk++) {
PetscInt lid = selected_idx[kk];
if (lid<nloc) {
ierr = MatGetRow(Gmat,lid+my0,&ncols,0,0);CHKERRQ(ierr);
if (ncols>1) clid_flid[nLocalSelected++] = lid; /* fiter out singletons */
ierr = MatRestoreRow(Gmat,lid+my0,&ncols,0,0);CHKERRQ(ierr);
}
}
ierr = ISRestoreIndices(selected_1, &selected_idx);CHKERRQ(ierr);
ierr = ISDestroy(&selected_1);CHKERRQ(ierr); /* this is selected_1 in serial */
/* create prolongator matrix */
ierr = MatGetType(Amat,&mtype);CHKERRQ(ierr);
ierr = MatCreate(comm, &Prol);CHKERRQ(ierr);
ierr = MatSetSizes(Prol,nloc*bs,nLocalSelected*bs,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatSetBlockSizes(Prol, bs, bs);CHKERRQ(ierr);
ierr = MatSetType(Prol, mtype);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(Prol,3*data_cols,NULL);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(Prol,3*data_cols,NULL,3*data_cols,NULL);CHKERRQ(ierr);
/* can get all points "removed" - but not on geomg */
ierr = MatGetSize(Prol, &kk, &jj);CHKERRQ(ierr);
if (!jj) {
ierr = PetscInfo(pc,"ERROE: no selected points on coarse grid\n");CHKERRQ(ierr);
ierr = PetscFree(clid_flid);CHKERRQ(ierr);
ierr = MatDestroy(&Prol);CHKERRQ(ierr);
*a_P_out = NULL; /* out */
PetscFunctionReturn(0);
}
{
PetscReal *coords;
PetscInt data_stride;
PetscInt *crsGID = NULL;
Mat Gmat2;
if (dim != data_cols) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_PLIB,"dim %D != data_cols %D",dim,data_cols);
/* grow ghost data for better coarse grid cover of fine grid */
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventBegin(petsc_gamg_setup_events[SET5],0,0,0,0);CHKERRQ(ierr);
#endif
/* messy method, squares graph and gets some data */
ierr = getGIDsOnSquareGraph(nLocalSelected, clid_flid, Gmat, &selected_2, &Gmat2, &crsGID);CHKERRQ(ierr);
/* llist is now not valid wrt squared graph, but will work as iterator in 'triangulateAndFormProl' */
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventEnd(petsc_gamg_setup_events[SET5],0,0,0,0);CHKERRQ(ierr);
#endif
/* create global vector of coorindates in 'coords' */
if (size > 1) {
ierr = PCGAMGGetDataWithGhosts(Gmat2, dim, pc_gamg->data, &data_stride, &coords);CHKERRQ(ierr);
} else {
coords = (PetscReal*)pc_gamg->data;
data_stride = pc_gamg->data_sz/pc_gamg->data_cell_cols;
}
ierr = MatDestroy(&Gmat2);CHKERRQ(ierr);
/* triangulate */
if (dim == 2) {
PetscReal metric,tm;
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventBegin(petsc_gamg_setup_events[SET6],0,0,0,0);CHKERRQ(ierr);
#endif
ierr = triangulateAndFormProl(selected_2, data_stride, coords,nLocalSelected, clid_flid, agg_lists, crsGID, bs, Prol, &metric);CHKERRQ(ierr);
#if defined PETSC_GAMG_USE_LOG
ierr = PetscLogEventEnd(petsc_gamg_setup_events[SET6],0,0,0,0);CHKERRQ(ierr);
#endif
ierr = PetscFree(crsGID);CHKERRQ(ierr);
/* clean up and create coordinates for coarse grid (output) */
if (size > 1) ierr = PetscFree(coords);CHKERRQ(ierr);
ierr = MPI_Allreduce(&metric, &tm, 1, MPIU_REAL, MPIU_MAX, comm);CHKERRQ(ierr);
if (tm > 1.) { /* needs to be globalized - should not happen */
ierr = PetscInfo1(pc," failed metric for coarse grid %e\n",(double)tm);CHKERRQ(ierr);
ierr = MatDestroy(&Prol);CHKERRQ(ierr);
} else if (metric > .0) {
ierr = PetscInfo1(pc,"worst metric for coarse grid = %e\n",(double)metric);CHKERRQ(ierr);
}
} else SETERRQ(comm,PETSC_ERR_PLIB,"3D not implemented for 'geo' AMG");
{ /* create next coords - output */
PetscReal *crs_crds;
ierr = PetscMalloc1(dim*nLocalSelected, &crs_crds);CHKERRQ(ierr);
for (kk=0; kk<nLocalSelected; kk++) { /* grab local select nodes to promote - output */
PetscInt lid = clid_flid[kk];
for (jj=0; jj<dim; jj++) crs_crds[jj*nLocalSelected + kk] = pc_gamg->data[jj*nloc + lid];
}
ierr = PetscFree(pc_gamg->data);CHKERRQ(ierr);
pc_gamg->data = crs_crds; /* out */
pc_gamg->data_sz = dim*nLocalSelected;
}
ierr = ISDestroy(&selected_2);CHKERRQ(ierr);
}
*a_P_out = Prol; /* out */
ierr = PetscFree(clid_flid);CHKERRQ(ierr);
ierr = PetscLogEventEnd(PC_GAMGProlongator_GEO,0,0,0,0);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
//==============================================================================
Epetra_PETScAIJMatrix::Epetra_PETScAIJMatrix(Mat Amat)
: Epetra_Object("Epetra::PETScAIJMatrix"),
Amat_(Amat),
Values_(0),
Indices_(0),
MaxNumEntries_(-1),
ImportVector_(0),
NormInf_(-1.0),
NormOne_(-1.0)
{
#ifdef HAVE_MPI
MPI_Comm comm;
PetscObjectGetComm( (PetscObject)Amat, &comm);
Comm_ = new Epetra_MpiComm(comm);
#else
Comm_ = new Epetra_SerialComm();
#endif
int ierr;
char errMsg[80];
MatGetType(Amat, &MatType_);
if ( strcmp(MatType_,MATSEQAIJ) != 0 && strcmp(MatType_,MATMPIAIJ) != 0 ) {
sprintf(errMsg,"PETSc matrix must be either seqaij or mpiaij (but it is %s)",MatType_);
throw Comm_->ReportError(errMsg,-1);
}
petscMatrixType mt;
Mat_MPIAIJ* aij=0;
if (strcmp(MatType_,MATMPIAIJ) == 0) {
mt = PETSC_MPI_AIJ;
aij = (Mat_MPIAIJ*)Amat->data;
}
else if (strcmp(MatType_,MATSEQAIJ) == 0) {
mt = PETSC_SEQ_AIJ;
}
int numLocalRows, numLocalCols;
ierr = MatGetLocalSize(Amat,&numLocalRows,&numLocalCols);
if (ierr) {
sprintf(errMsg,"EpetraExt_PETScAIJMatrix.cpp, line %d, MatGetLocalSize() returned error code %d",__LINE__,ierr);
throw Comm_->ReportError(errMsg,-1);
}
NumMyRows_ = numLocalRows;
NumMyCols_ = numLocalCols; //numLocalCols is the total # of unique columns in the local matrix (the diagonal block)
//TODO what happens if some columns are empty?
if (mt == PETSC_MPI_AIJ)
NumMyCols_ += aij->B->cmap->n;
MatInfo info;
ierr = MatGetInfo(Amat,MAT_LOCAL,&info);
if (ierr) {
sprintf(errMsg,"EpetraExt_PETScAIJMatrix.cpp, line %d, MatGetInfo() returned error code %d",__LINE__,ierr);
throw Comm_->ReportError(errMsg,-1);
}
NumMyNonzeros_ = (int) info.nz_used; //PETSc stores nnz as double
Comm_->SumAll(&(info.nz_used), &NumGlobalNonzeros_, 1);
//The PETSc documentation warns that this may not be robust.
//In particular, this will break if the ordering is not contiguous!
int rowStart, rowEnd;
ierr = MatGetOwnershipRange(Amat,&rowStart,&rowEnd);
if (ierr) {
sprintf(errMsg,"EpetraExt_PETScAIJMatrix.cpp, line %d, MatGetOwnershipRange() returned error code %d",__LINE__,ierr);
throw Comm_->ReportError(errMsg,-1);
}
PetscRowStart_ = rowStart;
PetscRowEnd_ = rowEnd;
int* MyGlobalElements = new int[rowEnd-rowStart];
for (int i=0; i<rowEnd-rowStart; i++)
MyGlobalElements[i] = rowStart+i;
ierr = MatGetInfo(Amat,MAT_GLOBAL_SUM,&info);
if (ierr) {
sprintf(errMsg,"EpetraExt_PETScAIJMatrix.cpp, line %d, MatGetInfo() returned error code %d",__LINE__,ierr);
throw Comm_->ReportError(errMsg,-1);
}
int tmp;
ierr = MatGetSize(Amat,&NumGlobalRows_,&tmp);
DomainMap_ = new Epetra_Map(NumGlobalRows_, NumMyRows_, MyGlobalElements, 0, *Comm_);
// get the GIDs of the non-local columns
//FIXME what if the matrix is sequential?
int * ColGIDs = new int[NumMyCols_];
for (int i=0; i<numLocalCols; i++) ColGIDs[i] = MyGlobalElements[i];
for (int i=numLocalCols; i<NumMyCols_; i++) ColGIDs[i] = aij->garray[i-numLocalCols];
ColMap_ = new Epetra_Map(-1, NumMyCols_, ColGIDs, 0, *Comm_);
Importer_ = new Epetra_Import(*ColMap_, *DomainMap_);
delete [] MyGlobalElements;
delete [] ColGIDs;
} //Epetra_PETScAIJMatrix(Mat Amat)
Modified from the code contributed by Yaning Liu @lbl.gov \n\n";
/*
Example:
mpiexec -n <np> ./ex103
mpiexec -n <np> ./ex103 -mat_type elemental -mat_view
mpiexec -n <np> ./ex103 -mat_type aij
*/
#include <petscmat.h>
#undef __FUNCT__
#define __FUNCT__ "main"
int main(int argc, char** argv)
{
Mat A,A_elemental;
PetscInt i,j,M=10,N=5,nrows,ncols;
PetscErrorCode ierr;
PetscMPIInt rank,size;
IS isrows,iscols;
const PetscInt *rows,*cols;
PetscScalar *v;
MatType type;
PetscBool isDense,isAIJ,flg;
ierr = PetscInitialize(&argc, &argv, (char*)0, help);if (ierr) return ierr;
#if !defined(PETSC_HAVE_ELEMENTAL)
SETERRQ(PETSC_COMM_WORLD,1,"This example requires ELEMENTAL");
#endif
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
/* Creat a matrix */
ierr = PetscOptionsGetInt(NULL,NULL,"-M",&M,NULL);CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-N",&N,NULL);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD, &A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,M,N);CHKERRQ(ierr);
ierr = MatSetType(A,MATDENSE);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
/* Set local matrix entries */
ierr = MatGetOwnershipIS(A,&isrows,&iscols);CHKERRQ(ierr);
ierr = ISGetLocalSize(isrows,&nrows);CHKERRQ(ierr);
ierr = ISGetIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISGetLocalSize(iscols,&ncols);CHKERRQ(ierr);
ierr = ISGetIndices(iscols,&cols);CHKERRQ(ierr);
ierr = PetscMalloc1(nrows*ncols,&v);CHKERRQ(ierr);
for (i=0; i<nrows; i++) {
for (j=0; j<ncols; j++) {
if (size == 1) {
v[i*ncols+j] = (PetscScalar)(i+j);
} else {
v[i*ncols+j] = (PetscScalar)rank+j*0.1;
}
}
}
ierr = MatSetValues(A,nrows,rows,ncols,cols,v,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
//ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%D] local nrows %D, ncols %D\n",rank,nrows,ncols);CHKERRQ(ierr);
//ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);CHKERRQ(ierr);
/* Test MatSetValues() by converting A to A_elemental */
ierr = MatGetType(A,&type);CHKERRQ(ierr);
if (size == 1) {
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQDENSE,&isDense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATSEQAIJ,&isAIJ);CHKERRQ(ierr);
} else {
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIDENSE,&isDense);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&isAIJ);CHKERRQ(ierr);
}
if (isDense || isAIJ) {
Mat Aexplicit;
ierr = MatConvert(A, MATELEMENTAL, MAT_INITIAL_MATRIX, &A_elemental);CHKERRQ(ierr);
ierr = MatComputeExplicitOperator(A_elemental,&Aexplicit);CHKERRQ(ierr);
ierr = MatMultEqual(Aexplicit,A_elemental,5,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Aexplicit != A_elemental.");
ierr = MatDestroy(&Aexplicit);CHKERRQ(ierr);
/* Test MAT_REUSE_MATRIX which is only supported for inplace conversion */
ierr = MatConvert(A, MATELEMENTAL, MAT_INPLACE_MATRIX, &A);CHKERRQ(ierr);
ierr = MatMultEqual(A_elemental,A,5,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"A_elemental != A.");
ierr = MatDestroy(&A_elemental);CHKERRQ(ierr);
}
ierr = ISRestoreIndices(isrows,&rows);CHKERRQ(ierr);
ierr = ISRestoreIndices(iscols,&cols);CHKERRQ(ierr);
ierr = ISDestroy(&isrows);CHKERRQ(ierr);
ierr = ISDestroy(&iscols);CHKERRQ(ierr);
ierr = PetscFree(v);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
// -------------------------------------------------------------
// MatMatMult_DenseGA
// -------------------------------------------------------------
static
PetscErrorCode
MatMatMult_DenseGA(Mat A, Mat B, MatReuse scall, PetscReal fill, Mat *C)
{
// matrix sizes appear to be checked before here, so we won't do it again
PetscErrorCode ierr = 0;
MPI_Comm comm;
ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr);
MatType atype, btype;
ierr = MatGetType(A, &atype);
ierr = MatGetType(B, &btype);
PetscBool issame;
PetscStrcmp(atype, btype, &issame);
Mat Bga, Cga;
struct MatGACtx *Actx, *Bctx, *Cctx;
ierr = MatShellGetContext(A, &Actx); CHKERRQ(ierr);
if (issame) {
Bga = B;
} else {
ierr = MatConvertToDenseGA(B, &Bga); CHKERRQ(ierr);
}
ierr = MatShellGetContext(Bga, &Bctx); CHKERRQ(ierr);
PetscInt lrows, lcols, grows, gcols, junk;
ierr = MatGetSize(A, &grows, &junk); CHKERRQ(ierr);
ierr = MatGetSize(B, &junk, &gcols); CHKERRQ(ierr);
ierr = MatGetLocalSize(A, &lrows, &junk); CHKERRQ(ierr);
ierr = MatGetLocalSize(B, &junk, &lcols); CHKERRQ(ierr);
ierr = MatCreateDenseGA(comm, lrows, lcols, grows, gcols, &Cga); CHKERRQ(ierr);
ierr = MatShellGetContext(Cga, &Cctx); CHKERRQ(ierr);
PetscScalarGA alpha(one), beta(zero);
int ndim, itype, lo[2] = {0,0}, ahi[2], bhi[2], chi[2];
NGA_Inquire(Actx->ga, &itype, &ndim, ahi);
ahi[0] -= 1; ahi[1] -= 1;
NGA_Inquire(Bctx->ga, &itype, &ndim, bhi);
bhi[0] -= 1; bhi[1] -= 1;
NGA_Inquire(Cctx->ga, &itype, &ndim, chi);
chi[0] -= 1; chi[1] -= 1;
// GA_Print(Actx->ga);
// GA_Print(Bctx->ga);
NGA_Matmul_patch('N', 'N', &alpha, &beta,
Actx->ga, lo, ahi,
Bctx->ga, lo, bhi,
Cctx->ga, lo, chi);
// GA_Print(Cctx->ga);
switch (scall) {
case MAT_REUSE_MATRIX:
ierr = MatCopy(Cga, *C, SAME_NONZERO_PATTERN); CHKERRQ(ierr);
case MAT_INITIAL_MATRIX:
default:
ierr = MatDuplicate(Cga, MAT_COPY_VALUES, C); CHKERRQ(ierr);
break;
}
if (!issame) ierr = MatDestroy(&Bga); CHKERRQ(ierr);
ierr = MatDestroy(&Cga); CHKERRQ(ierr);
return ierr;
}
PetscErrorCode PCGAMGGraph_Classical(PC pc,const Mat A,Mat *G)
{
PetscInt s,f,n,idx,lidx,gidx;
PetscInt r,c,ncols;
const PetscInt *rcol;
const PetscScalar *rval;
PetscInt *gcol;
PetscScalar *gval;
PetscReal rmax;
PetscInt cmax = 0;
PC_MG *mg;
PC_GAMG *gamg;
PetscErrorCode ierr;
PetscInt *gsparse,*lsparse;
PetscScalar *Amax;
MatType mtype;
PetscFunctionBegin;
mg = (PC_MG *)pc->data;
gamg = (PC_GAMG *)mg->innerctx;
ierr = MatGetOwnershipRange(A,&s,&f);CHKERRQ(ierr);
n=f-s;
ierr = PetscMalloc(sizeof(PetscInt)*n,&lsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*n,&gsparse);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscScalar)*n,&Amax);CHKERRQ(ierr);
for (r = 0;r < n;r++) {
lsparse[r] = 0;
gsparse[r] = 0;
}
for (r = s;r < f;r++) {
/* determine the maximum off-diagonal in each row */
rmax = 0.;
ierr = MatGetRow(A,r,&ncols,&rcol,&rval);CHKERRQ(ierr);
for (c = 0; c < ncols; c++) {
if (PetscRealPart(-rval[c]) > rmax && rcol[c] != r) {
rmax = PetscRealPart(-rval[c]);
}
}
Amax[r-s] = rmax;
if (ncols > cmax) cmax = ncols;
lidx = 0;
gidx = 0;
/* create the local and global sparsity patterns */
for (c = 0; c < ncols; c++) {
if (PetscRealPart(-rval[c]) > gamg->threshold*PetscRealPart(Amax[r-s]) || rcol[c] == r) {
if (rcol[c] < f && rcol[c] >= s) {
lidx++;
} else {
gidx++;
}
}
}
ierr = MatRestoreRow(A,r,&ncols,&rcol,&rval);CHKERRQ(ierr);
lsparse[r-s] = lidx;
gsparse[r-s] = gidx;
}
ierr = PetscMalloc(sizeof(PetscScalar)*cmax,&gval);CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(PetscInt)*cmax,&gcol);CHKERRQ(ierr);
ierr = MatCreate(PetscObjectComm((PetscObject)A),G); CHKERRQ(ierr);
ierr = MatGetType(A,&mtype);CHKERRQ(ierr);
ierr = MatSetType(*G,mtype);CHKERRQ(ierr);
ierr = MatSetSizes(*G,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
ierr = MatMPIAIJSetPreallocation(*G,0,lsparse,0,gsparse);CHKERRQ(ierr);
ierr = MatSeqAIJSetPreallocation(*G,0,lsparse);CHKERRQ(ierr);
for (r = s;r < f;r++) {
ierr = MatGetRow(A,r,&ncols,&rcol,&rval);CHKERRQ(ierr);
idx = 0;
for (c = 0; c < ncols; c++) {
/* classical strength of connection */
if (PetscRealPart(-rval[c]) > gamg->threshold*PetscRealPart(Amax[r-s]) || rcol[c] == r) {
gcol[idx] = rcol[c];
gval[idx] = rval[c];
idx++;
}
}
ierr = MatSetValues(*G,1,&r,idx,gcol,gval,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatRestoreRow(A,r,&ncols,&rcol,&rval);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(*G, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(*G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = PetscFree(gval);CHKERRQ(ierr);
ierr = PetscFree(gcol);CHKERRQ(ierr);
ierr = PetscFree(lsparse);CHKERRQ(ierr);
ierr = PetscFree(gsparse);CHKERRQ(ierr);
ierr = PetscFree(Amax);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **args)
{
Vec x,y,u,s1,s2;
Mat A,sA,sB;
PetscRandom rctx;
PetscReal r1,r2,rnorm,tol = PETSC_SQRT_MACHINE_EPSILON;
PetscScalar one=1.0, neg_one=-1.0, value[3], four=4.0,alpha=0.1;
PetscInt n,col[3],n1,block,row,i,j,i2,j2,Ii,J,rstart,rend,bs=1,mbs=16,d_nz=3,o_nz=3,prob=2;
PetscErrorCode ierr;
PetscMPIInt size,rank;
PetscBool flg;
MatType type;
ierr = PetscInitialize(&argc,&args,(char*)0,help);
if (ierr) return ierr;
ierr = PetscOptionsGetInt(NULL,NULL,"-mbs",&mbs,NULL);
CHKERRQ(ierr);
ierr = PetscOptionsGetInt(NULL,NULL,"-bs",&bs,NULL);
CHKERRQ(ierr);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
CHKERRQ(ierr);
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);
CHKERRQ(ierr);
n = mbs*bs;
/* Assemble MPISBAIJ matrix sA */
ierr = MatCreate(PETSC_COMM_WORLD,&sA);
CHKERRQ(ierr);
ierr = MatSetSizes(sA,PETSC_DECIDE,PETSC_DECIDE,n,n);
CHKERRQ(ierr);
ierr = MatSetType(sA,MATSBAIJ);
CHKERRQ(ierr);
ierr = MatSetFromOptions(sA);
CHKERRQ(ierr);
ierr = MatGetType(sA,&type);
CHKERRQ(ierr);
ierr = MatMPISBAIJSetPreallocation(sA,bs,d_nz,NULL,o_nz,NULL);
CHKERRQ(ierr);
ierr = MatSeqSBAIJSetPreallocation(sA,bs,d_nz,NULL);
CHKERRQ(ierr);
ierr = MatSetOption(sA,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);
CHKERRQ(ierr);
if (bs == 1) {
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0;
value[1] = 2.0;
value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = n - 1;
col[0]=0;
col[1] = n - 2;
col[2] = n - 1;
value[0]= 0.1;
value[1]=-1;
value[2]=2;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0;
col[0] = 0;
col[1] = 1;
col[2]=n-1;
value[0] = 2.0;
value[1] = -1.0;
value[2]=0.1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (int) PetscSqrtReal((PetscReal)n);
if (n1*n1 != n) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"n must be a perfect square of n1");
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(sA,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
ierr = MatSetValues(sA,1,&Ii,1,&Ii,&four,INSERT_VALUES);
CHKERRQ(ierr);
}
}
}
/* end of if (bs == 1) */
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0;
value[1] = 4.0;
value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(sA,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = bs - 1+block*bs;
col[0] = bs - 2+block*bs;
col[1] = bs - 1+block*bs;
value[0]=-1.0;
value[1]=4.0;
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0+block*bs;
col[0] = 0+block*bs;
col[1] = 1+block*bs;
value[0]=4.0;
value[1] = -1.0;
ierr = MatSetValues(sA,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(sA,1,&i,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
col[0]=i;
row=i+bs;
ierr = MatSetValues(sA,1,&row,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(sA,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(sA,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
/* Test MatView() */
ierr = MatCreateBAIJ(PETSC_COMM_WORLD,bs,PETSC_DECIDE,PETSC_DECIDE,n,n,d_nz,NULL,o_nz,NULL,&A);
CHKERRQ(ierr);
ierr = MatSetOption(A,MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_FALSE);
CHKERRQ(ierr);
if (bs == 1) {
if (prob == 1) { /* tridiagonal matrix */
value[0] = -1.0;
value[1] = 2.0;
value[2] = -1.0;
for (i=1; i<n-1; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = n - 1;
col[0]=0;
col[1] = n - 2;
col[2] = n - 1;
value[0]= 0.1;
value[1]=-1;
value[2]=2;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0;
col[0] = 0;
col[1] = 1;
col[2]=n-1;
value[0] = 2.0;
value[1] = -1.0;
value[2]=0.1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
} else if (prob ==2) { /* matrix for the five point stencil */
n1 = (int) PetscSqrtReal((PetscReal)n);
for (i=0; i<n1; i++) {
for (j=0; j<n1; j++) {
Ii = j + n1*i;
if (i>0) {
J = Ii - n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (i<n1-1) {
J = Ii + n1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j>0) {
J = Ii - 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
if (j<n1-1) {
J = Ii + 1;
ierr = MatSetValues(A,1,&Ii,1,&J,&neg_one,INSERT_VALUES);
CHKERRQ(ierr);
}
ierr = MatSetValues(A,1,&Ii,1,&Ii,&four,INSERT_VALUES);
CHKERRQ(ierr);
}
}
}
/* end of if (bs == 1) */
} else { /* bs > 1 */
for (block=0; block<n/bs; block++) {
/* diagonal blocks */
value[0] = -1.0;
value[1] = 4.0;
value[2] = -1.0;
for (i=1+block*bs; i<bs-1+block*bs; i++) {
col[0] = i-1;
col[1] = i;
col[2] = i+1;
ierr = MatSetValues(A,1,&i,3,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
i = bs - 1+block*bs;
col[0] = bs - 2+block*bs;
col[1] = bs - 1+block*bs;
value[0]=-1.0;
value[1]=4.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
i = 0+block*bs;
col[0] = 0+block*bs;
col[1] = 1+block*bs;
value[0]=4.0;
value[1] = -1.0;
ierr = MatSetValues(A,1,&i,2,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
/* off-diagonal blocks */
value[0]=-1.0;
for (i=0; i<(n/bs-1)*bs; i++) {
col[0]=i+bs;
ierr = MatSetValues(A,1,&i,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
col[0]=i;
row=i+bs;
ierr = MatSetValues(A,1,&row,1,col,value,INSERT_VALUES);
CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
CHKERRQ(ierr);
/* Test MatGetSize(), MatGetLocalSize() */
ierr = MatGetSize(sA, &i,&j);
CHKERRQ(ierr);
ierr = MatGetSize(A, &i2,&j2);
CHKERRQ(ierr);
i -= i2;
j -= j2;
if (i || j) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatGetSize()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatGetLocalSize(sA, &i,&j);
CHKERRQ(ierr);
ierr = MatGetLocalSize(A, &i2,&j2);
CHKERRQ(ierr);
i2 -= i;
j2 -= j;
if (i2 || j2) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatGetLocalSize()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* vectors */
/*--------------------*/
/* i is obtained from MatGetLocalSize() */
ierr = VecCreate(PETSC_COMM_WORLD,&x);
CHKERRQ(ierr);
ierr = VecSetSizes(x,i,PETSC_DECIDE);
CHKERRQ(ierr);
ierr = VecSetFromOptions(x);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&y);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&u);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&s1);
CHKERRQ(ierr);
ierr = VecDuplicate(x,&s2);
CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);
CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);
CHKERRQ(ierr);
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = VecSet(u,one);
CHKERRQ(ierr);
/* Test MatNorm() */
ierr = MatNorm(A,NORM_FROBENIUS,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_FROBENIUS,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatNorm_FROBENIUS(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_INFINITY,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_INFINITY,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: MatNorm_INFINITY(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
ierr = MatNorm(A,NORM_1,&r1);
CHKERRQ(ierr);
ierr = MatNorm(sA,NORM_1,&r2);
CHKERRQ(ierr);
rnorm = PetscAbsReal(r1-r2)/r2;
if (rnorm > tol && !rank) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: MatNorm_1(), Anorm=%16.14e, sAnorm=%16.14e bs=%D\n",r1,r2,bs);
CHKERRQ(ierr);
}
/* Test MatGetOwnershipRange() */
ierr = MatGetOwnershipRange(sA,&rstart,&rend);
CHKERRQ(ierr);
ierr = MatGetOwnershipRange(A,&i2,&j2);
CHKERRQ(ierr);
i2 -= rstart;
j2 -= rend;
if (i2 || j2) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MaGetOwnershipRange()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* Test MatDiagonalScale() */
ierr = MatDiagonalScale(A,x,x);
CHKERRQ(ierr);
ierr = MatDiagonalScale(sA,x,x);
CHKERRQ(ierr);
ierr = MatMultEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMETYPE,"Error in MatDiagonalScale");
/* Test MatGetDiagonal(), MatScale() */
ierr = MatGetDiagonal(A,s1);
CHKERRQ(ierr);
ierr = MatGetDiagonal(sA,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDiagonalScale() or MatGetDiagonal(), r1=%g \n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatScale(A,alpha);
CHKERRQ(ierr);
ierr = MatScale(sA,alpha);
CHKERRQ(ierr);
/* Test MatGetRowMaxAbs() */
ierr = MatGetRowMaxAbs(A,s1,NULL);
CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(sA,s2,NULL);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscPrintf(PETSC_COMM_SELF,"Error: MatGetRowMaxAbs() \n");
CHKERRQ(ierr);
}
/* Test MatMult(), MatMultAdd() */
ierr = MatMultEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMult() or MatScale()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatMultAddEqual(A,sA,10,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMultAdd()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
/* Test MatMultTranspose(), MatMultTransposeAdd() */
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = MatMultTranspose(A,x,s1);
CHKERRQ(ierr);
ierr = MatMultTranspose(sA,x,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMult() or MatScale(), err=%g\n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
}
for (i=0; i<10; i++) {
ierr = VecSetRandom(x,rctx);
CHKERRQ(ierr);
ierr = VecSetRandom(y,rctx);
CHKERRQ(ierr);
ierr = MatMultTransposeAdd(A,x,y,s1);
CHKERRQ(ierr);
ierr = MatMultTransposeAdd(sA,x,y,s2);
CHKERRQ(ierr);
ierr = VecNorm(s1,NORM_1,&r1);
CHKERRQ(ierr);
ierr = VecNorm(s2,NORM_1,&r2);
CHKERRQ(ierr);
r1 -= r2;
if (r1<-tol || r1>tol) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatMultAdd(), err=%g \n",rank,(double)r1);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
}
/* Test MatDuplicate() */
ierr = MatDuplicate(sA,MAT_COPY_VALUES,&sB);
CHKERRQ(ierr);
ierr = MatEqual(sA,sB,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD," Error in MatDuplicate(), sA != sB \n");
CHKERRQ(ierr);
CHKERRQ(ierr);
}
ierr = MatMultEqual(sA,sB,5,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDuplicate() or MatMult()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatMultAddEqual(sA,sB,5,&flg);
CHKERRQ(ierr);
if (!flg) {
ierr = PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d], Error: MatDuplicate() or MatMultAdd(()\n",rank);
CHKERRQ(ierr);
ierr = PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
CHKERRQ(ierr);
}
ierr = MatDestroy(&sB);
CHKERRQ(ierr);
ierr = VecDestroy(&u);
CHKERRQ(ierr);
ierr = VecDestroy(&x);
CHKERRQ(ierr);
ierr = VecDestroy(&y);
CHKERRQ(ierr);
ierr = VecDestroy(&s1);
CHKERRQ(ierr);
ierr = VecDestroy(&s2);
CHKERRQ(ierr);
ierr = MatDestroy(&sA);
CHKERRQ(ierr);
ierr = MatDestroy(&A);
CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);
CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
int main(int argc,char **args)
{
Mat A;
Vec min,max,maxabs;
PetscInt m,n;
PetscInt imin[M],imax[M],imaxabs[M],indices[N],row;
PetscScalar values[N];
PetscErrorCode ierr;
MatType type;
PetscMPIInt size;
PetscBool doTest=PETSC_TRUE;
ierr = PetscInitialize(&argc,&args,(char*)0,help);if (ierr) return ierr;
ierr = MPI_Comm_size(PETSC_COMM_WORLD,&size);CHKERRQ(ierr);
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,M,N);CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
ierr = MatSetUp(A);CHKERRQ(ierr);
row = 0;
indices[0] = 0; indices[1] = 1; indices[2] = 2; indices[3] = 3; indices[4] = 4; indices[5] = 5;
values[0] = -1.0; values[1] = 0.0; values[2] = 1.0; values[3] = 3.0; values[4] = 4.0; values[5] = -5.0;
ierr = MatSetValues(A,1,&row,6,indices,values,INSERT_VALUES);CHKERRQ(ierr);
row = 1;
ierr = MatSetValues(A,1,&row,3,indices,values,INSERT_VALUES);CHKERRQ(ierr);
row = 4;
ierr = MatSetValues(A,1,&row,1,indices+4,values+4,INSERT_VALUES);CHKERRQ(ierr);
row = 4;
ierr = MatSetValues(A,1,&row,2,indices+4,values+4,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = MatGetLocalSize(A, &m,&n);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_WORLD,&min);CHKERRQ(ierr);
ierr = VecSetSizes(min,m,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetFromOptions(min);CHKERRQ(ierr);
ierr = VecDuplicate(min,&max);CHKERRQ(ierr);
ierr = VecDuplicate(min,&maxabs);CHKERRQ(ierr);
/* Test MatGetRowMin, MatGetRowMax and MatGetRowMaxAbs */
if (size == 1) {
ierr = MatGetRowMin(A,min,imin);CHKERRQ(ierr);
ierr = MatGetRowMax(A,max,imax);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Minimums\n");CHKERRQ(ierr);
ierr = VecView(min,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximums\n");CHKERRQ(ierr);
ierr = VecView(max,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imax,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values\n");CHKERRQ(ierr);
ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imaxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
} else {
ierr = MatGetType(A,&type);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nMatrix type: %s\n",type);CHKERRQ(ierr);
/* AIJ */
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&doTest);CHKERRQ(ierr);
if (doTest) {
ierr = MatGetRowMaxAbs(A,maxabs,NULL);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values:\n");CHKERRQ(ierr);
ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
/* BAIJ */
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIBAIJ,&doTest);CHKERRQ(ierr);
if (doTest) {
ierr = MatGetRowMaxAbs(A,maxabs,NULL);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values:\n");CHKERRQ(ierr);
ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
}
if (size == 1) {
ierr = MatConvert(A,MATDENSE,MAT_INPLACE_MATRIX,&A);CHKERRQ(ierr);
ierr = MatGetRowMin(A,min,imin);CHKERRQ(ierr);
ierr = MatGetRowMax(A,max,imax);CHKERRQ(ierr);
ierr = MatGetRowMaxAbs(A,maxabs,imaxabs);CHKERRQ(ierr);
ierr = MatView(A,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Minimums\n");CHKERRQ(ierr);
ierr = VecView(min,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imin,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximums\n");CHKERRQ(ierr);
ierr = VecView(max,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imax,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Row Maximum Absolute Values\n");CHKERRQ(ierr);
ierr = VecView(maxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscIntView(5,imaxabs,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = VecDestroy(&min);CHKERRQ(ierr);
ierr = VecDestroy(&max);CHKERRQ(ierr);
ierr = VecDestroy(&maxabs);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = PetscFinalize();
return ierr;
}
/*
I should not modify setup called!!
This is handled via petsc.
*/
PetscErrorCode BSSCR_PCSetUp_GtKG( PC pc )
{
PC_GtKG ctx = (PC_GtKG)pc->data;
PetscReal fill;
Mat Ident;
Vec diag;
PetscInt M,N, m,n;
MPI_Comm comm;
PetscInt nnz_I, nnz_G;
const MatType mtype;
const char *prefix;
PetscTruth wasSetup;
if( ctx->K == PETSC_NULL ) {
Stg_SETERRQ( PETSC_ERR_SUP, "gtkg: K not set" );
}
if( ctx->G == PETSC_NULL ) {
Stg_SETERRQ( PETSC_ERR_SUP, "gtkg: G not set" );
}
PetscObjectGetComm( (PetscObject)ctx->K, &comm );
/* Check for existence of objects and trash any which exist */
if( ctx->form_GtG == PETSC_TRUE && ctx->GtG != PETSC_NULL ) {
Stg_MatDestroy(&ctx->GtG );
ctx->GtG = PETSC_NULL;
}
if( ctx->s != PETSC_NULL ) {
Stg_VecDestroy(&ctx->s );
ctx->s = PETSC_NULL;
}
if( ctx->X != PETSC_NULL ) {
Stg_VecDestroy(&ctx->X );
ctx->X = PETSC_NULL;
}
if( ctx->t != PETSC_NULL ) {
Stg_VecDestroy(&ctx->t );
ctx->t = PETSC_NULL;
}
if( ctx->inv_diag_M != PETSC_NULL ) {
Stg_VecDestroy(&ctx->inv_diag_M );
ctx->inv_diag_M = PETSC_NULL;
}
/* Create vectors */
MatGetVecs( ctx->K, &ctx->s, &ctx->X );
MatGetVecs( ctx->G, &ctx->t, PETSC_NULL );
if( ctx->M != PETSC_NULL ) {
MatGetVecs( ctx->K, &ctx->inv_diag_M, PETSC_NULL );
MatGetDiagonal( ctx->M, ctx->inv_diag_M );
VecReciprocal( ctx->inv_diag_M );
/* change the pc_apply routines */
pc->ops->apply = BSSCR_BSSCR_PCApply_GtKG_diagonal_scaling;
pc->ops->applytranspose = BSSCR_BSSCR_PCApplyTranspose_GtKG_diagonal_scaling;
}
/* Assemble GtG */
MatGetSize( ctx->G, &M, &N );
MatGetLocalSize( ctx->G, &m, &n );
MatGetVecs( ctx->G, PETSC_NULL, &diag );
VecSet( diag, 1.0 );
MatCreate( comm, &Ident );
MatSetSizes( Ident, m,m , M, M );
#if (((PETSC_VERSION_MAJOR==3) && (PETSC_VERSION_MINOR>=3)) || (PETSC_VERSION_MAJOR>3) )
MatSetUp(Ident);
#endif
MatGetType( ctx->G, &mtype );
MatSetType( Ident, mtype );
if( ctx->M == PETSC_NULL ) {
MatDiagonalSet( Ident, diag, INSERT_VALUES );
}
else {
MatDiagonalSet( Ident, ctx->inv_diag_M, INSERT_VALUES );
}
BSSCR_get_number_nonzeros_AIJ( Ident, &nnz_I );
BSSCR_get_number_nonzeros_AIJ( ctx->G, &nnz_G );
//fill = 1.0;
/*
Not sure the best way to estimate the fill factor.
GtG is a laplacian on the pressure space.
This might tell us something useful...
*/
fill = (PetscReal)(nnz_G)/(PetscReal)( nnz_I );
MatPtAP( Ident, ctx->G, MAT_INITIAL_MATRIX, fill, &ctx->GtG );
Stg_MatDestroy(&Ident);
Stg_VecDestroy(&diag );
Stg_KSPSetOperators( ctx->ksp, ctx->GtG, ctx->GtG, SAME_NONZERO_PATTERN );
if (!pc->setupcalled) {
wasSetup = PETSC_FALSE;
PCGetOptionsPrefix( pc,&prefix );
KSPSetOptionsPrefix( ctx->ksp, prefix );
KSPAppendOptionsPrefix( ctx->ksp, "pc_gtkg_" ); /* -pc_GtKG_ksp_type <type>, -ksp_GtKG_pc_type <type> */
}
else {
wasSetup = PETSC_TRUE;
}
// if (!wasSetup && pc->setfromoptionscalled) {
if (!wasSetup) {
KSPSetFromOptions(ctx->ksp);
}
PetscFunctionReturn(0);
}
PetscErrorCode _BlockSolve( void* solver, void* _stokesSLE ) {
Stokes_SLE* stokesSLE = (Stokes_SLE*)_stokesSLE;
StokesBlockKSPInterface* Solver = (StokesBlockKSPInterface*)solver;
/* Create shortcuts to stuff needed on sle */
Mat K;
Mat G;
Mat Gt;
Mat D;
Mat C;
Mat approxS;
Vec u;
Vec p;
Vec f;
Vec h;
Mat stokes_P;
Mat stokes_A;
Vec stokes_x;
Vec stokes_b;
Mat a[2][2];
Vec x[2];
Vec b[2];
KSP stokes_ksp;
PC stokes_pc;
PetscTruth sym,flg;
PetscErrorCode ierr;
PetscInt N,n;
SBKSP_GetStokesOperators( stokesSLE, &K,&G,&D,&C, &approxS, &f,&h, &u,&p );
/* create Gt */
if( !D ) {
ierr = MatTranspose( G, MAT_INITIAL_MATRIX, &Gt);CHKERRQ(ierr);
sym = PETSC_TRUE;
Solver->DIsSym = sym;
}
else {
Gt = D;
sym = PETSC_FALSE;
Solver->DIsSym = sym;
}
flg=PETSC_FALSE;
PetscOptionsHasName(PETSC_NULL,"-use_petsc_ksp",&flg);
if (flg) {
if( !C ) {
/* Everything in this bracket, dependent on !C, is to build
a matrix with diagonals of 0 for C the previous comment ways
need a 'zero' matrix to keep fieldsplit happy in petsc? */
MatType mtype;
Vec V;
//MatGetSize( G, &M, &N );
VecGetSize(p, &N);
VecGetLocalSize( p, &n );
MatCreate( PetscObjectComm((PetscObject) K), &C );
MatSetSizes( C, PETSC_DECIDE ,PETSC_DECIDE, N, N );
#if (((PETSC_VERSION_MAJOR==3) && (PETSC_VERSION_MINOR>=3)) || (PETSC_VERSION_MAJOR>3) )
MatSetUp(C);
#endif
MatGetType( G, &mtype );
MatSetType( C, mtype );
MatGetVecs( G, &V, PETSC_NULL );
VecSet(V, 0.0);
//VecSet(h, 1.0);
ierr = VecAssemblyBegin( V );CHKERRQ(ierr);
ierr = VecAssemblyEnd ( V );CHKERRQ(ierr);
ierr = MatDiagonalSet(C,V,INSERT_VALUES);CHKERRQ(ierr);
ierr = MatAssemblyBegin( C, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr);
ierr = MatAssemblyEnd ( C, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr);
}
}
a[0][0]=K; a[0][1]=G;
a[1][0]=Gt; a[1][1]=C;
ierr = MatCreateNest(PetscObjectComm((PetscObject) K), 2, NULL, 2, NULL, (Mat *)a, &stokes_A);CHKERRQ(ierr);
ierr = MatAssemblyBegin( stokes_A, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr);
ierr = MatAssemblyEnd( stokes_A, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr);
x[0]=u;
x[1]=p;
ierr = VecCreateNest(PetscObjectComm((PetscObject) u), 2, NULL, x, &stokes_x);CHKERRQ(ierr);
ierr = VecAssemblyBegin( stokes_x );CHKERRQ(ierr);
ierr = VecAssemblyEnd( stokes_x);CHKERRQ(ierr);
b[0]=f;
b[1]=h;
ierr = VecCreateNest(PetscObjectComm((PetscObject) f), 2, NULL, b, &stokes_b);CHKERRQ(ierr);
ierr = VecAssemblyBegin( stokes_b );CHKERRQ(ierr);
ierr = VecAssemblyEnd( stokes_b);CHKERRQ(ierr);
/* if( approxS ) { */
/* a[0][0]=K; a[0][1]=G; */
/* a[1][0]=NULL; a[1][1]=approxS; */
/* ierr = MatCreateNest(PetscObjectComm((PetscObject) K), 2, NULL, 2, NULL, (Mat *)a, &stokes_P);CHKERRQ(ierr); */
/* ierr = MatAssemblyBegin( stokes_P, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr); */
/* ierr = MatAssemblyEnd( stokes_P, MAT_FINAL_ASSEMBLY );CHKERRQ(ierr); */
/* } */
/* else { */
stokes_P = stokes_A;
/* } */
/* probably should make a Destroy function for these two */
/* Update options from file and/or string here so we can change things on the fly */
//PetscOptionsInsertFile(PETSC_COMM_WORLD, Solver->optionsFile, PETSC_FALSE);
//PetscOptionsInsertString(Solver->optionsString);
ierr = KSPCreate( PETSC_COMM_WORLD, &stokes_ksp );CHKERRQ(ierr);
Stg_KSPSetOperators( stokes_ksp, stokes_A, stokes_P, SAME_NONZERO_PATTERN );
ierr = KSPSetType( stokes_ksp, "bsscr" );/* i.e. making this the default solver : calls KSPCreate_XXX */CHKERRQ(ierr);
ierr = KSPGetPC( stokes_ksp, &stokes_pc );CHKERRQ(ierr);
ierr = PCSetType( stokes_pc, PCNONE );CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero( stokes_ksp, PETSC_TRUE );CHKERRQ(ierr);
ierr = KSPSetFromOptions( stokes_ksp );CHKERRQ(ierr);
/*
Doing this so the KSP Solver has access to the StgFEM Multigrid struct (PETScMGSolver).
As well as any custom stuff on the Stokes_SLE struct
*/
if( stokes_ksp->data ){/* then ksp->data has been created in a KSpSetUp_XXX function */
/* testing for our KSP types that need the data that is on Solver... */
/* for the moment then, this function not completely agnostic about our KSPs */
//if(!strcmp("bsscr",stokes_ksp->type_name)){/* if is bsscr then set up the data on the ksp */
flg=PETSC_FALSE;
PetscOptionsHasName(PETSC_NULL,"-use_petsc_ksp",&flg);
if (!flg) {
((KSP_COMMON*)(stokes_ksp->data))->st_sle = Solver->st_sle;
((KSP_COMMON*)(stokes_ksp->data))->mg = Solver->mg;
((KSP_COMMON*)(stokes_ksp->data))->DIsSym = Solver->DIsSym;
((KSP_COMMON*)(stokes_ksp->data))->preconditioner = Solver->preconditioner;
((KSP_COMMON*)(stokes_ksp->data))->solver = Solver;
}
}
ierr = KSPSolve( stokes_ksp, stokes_b, stokes_x );CHKERRQ(ierr);
Stg_KSPDestroy(&stokes_ksp );
//if( ((StokesBlockKSPInterface*)stokesSLE->solver)->preconditioner )
if(stokes_P != stokes_A) { Stg_MatDestroy(&stokes_P ); }
Stg_MatDestroy(&stokes_A );
Stg_VecDestroy(&stokes_x);
Stg_VecDestroy(&stokes_b);
if(!D){ Stg_MatDestroy(&Gt); }
if(C && (stokesSLE->cStiffMat->matrix != C) ){ Stg_MatDestroy(&C); }
PetscFunctionReturn(0);
}