PetscErrorCode PCSetUp_ML(PC pc)
{
PetscErrorCode ierr;
PetscMPIInt size;
FineGridCtx *PetscMLdata;
ML *ml_object;
ML_Aggregate *agg_object;
ML_Operator *mlmat;
PetscInt nlocal_allcols,Nlevels,mllevel,level,level1,m,fine_level,bs;
Mat A,Aloc;
GridCtx *gridctx;
PC_MG *mg = (PC_MG*)pc->data;
PC_ML *pc_ml = (PC_ML*)mg->innerctx;
PetscBool isSeq, isMPI;
KSP smoother;
PC subpc;
PetscInt mesh_level, old_mesh_level;
PetscFunctionBegin;
A = pc->pmat;
ierr = MPI_Comm_size(((PetscObject)A)->comm,&size);CHKERRQ(ierr);
if (pc->setupcalled) {
if (pc->flag == SAME_NONZERO_PATTERN && pc_ml->reuse_interpolation) {
/*
Reuse interpolaton instead of recomputing aggregates and updating the whole hierarchy. This is less expensive for
multiple solves in which the matrix is not changing too quickly.
*/
ml_object = pc_ml->ml_object;
gridctx = pc_ml->gridctx;
Nlevels = pc_ml->Nlevels;
fine_level = Nlevels - 1;
gridctx[fine_level].A = A;
ierr = PetscObjectTypeCompare((PetscObject) A, MATSEQAIJ, &isSeq);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject) A, MATMPIAIJ, &isMPI);CHKERRQ(ierr);
if (isMPI){
ierr = MatConvert_MPIAIJ_ML(A,PETSC_NULL,MAT_INITIAL_MATRIX,&Aloc);CHKERRQ(ierr);
} else if (isSeq) {
Aloc = A;
ierr = PetscObjectReference((PetscObject)Aloc);CHKERRQ(ierr);
} else SETERRQ1(((PetscObject)pc)->comm,PETSC_ERR_ARG_WRONG, "Matrix type '%s' cannot be used with ML. ML can only handle AIJ matrices.",((PetscObject)A)->type_name);
ierr = MatGetSize(Aloc,&m,&nlocal_allcols);CHKERRQ(ierr);
PetscMLdata = pc_ml->PetscMLdata;
ierr = MatDestroy(&PetscMLdata->Aloc);CHKERRQ(ierr);
PetscMLdata->A = A;
PetscMLdata->Aloc = Aloc;
ML_Init_Amatrix(ml_object,0,m,m,PetscMLdata);
ML_Set_Amatrix_Matvec(ml_object,0,PetscML_matvec);
mesh_level = ml_object->ML_finest_level;
while (ml_object->SingleLevel[mesh_level].Rmat->to) {
old_mesh_level = mesh_level;
mesh_level = ml_object->SingleLevel[mesh_level].Rmat->to->levelnum;
/* clean and regenerate A */
mlmat = &(ml_object->Amat[mesh_level]);
ML_Operator_Clean(mlmat);
ML_Operator_Init(mlmat,ml_object->comm);
ML_Gen_AmatrixRAP(ml_object, old_mesh_level, mesh_level);
}
level = fine_level - 1;
if (size == 1) { /* convert ML P, R and A into seqaij format */
for (mllevel=1; mllevel<Nlevels; mllevel++){
mlmat = &(ml_object->Amat[mllevel]);
ierr = MatWrapML_SeqAIJ(mlmat,MAT_REUSE_MATRIX,&gridctx[level].A);CHKERRQ(ierr);
level--;
}
} else { /* convert ML P and R into shell format, ML A into mpiaij format */
for (mllevel=1; mllevel<Nlevels; mllevel++){
mlmat = &(ml_object->Amat[mllevel]);
ierr = MatWrapML_MPIAIJ(mlmat,MAT_REUSE_MATRIX,&gridctx[level].A);CHKERRQ(ierr);
level--;
}
}
for (level=0; level<fine_level; level++) {
if (level > 0){
ierr = PCMGSetResidual(pc,level,PCMGDefaultResidual,gridctx[level].A);CHKERRQ(ierr);
}
ierr = KSPSetOperators(gridctx[level].ksp,gridctx[level].A,gridctx[level].A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
}
ierr = PCMGSetResidual(pc,fine_level,PCMGDefaultResidual,gridctx[fine_level].A);CHKERRQ(ierr);
ierr = KSPSetOperators(gridctx[fine_level].ksp,gridctx[level].A,gridctx[fine_level].A,SAME_NONZERO_PATTERN);CHKERRQ(ierr);
ierr = PCSetUp_MG(pc);CHKERRQ(ierr);
PetscFunctionReturn(0);
} else {
/* since ML can change the size of vectors/matrices at any level we must destroy everything */
ierr = PCReset_ML(pc);CHKERRQ(ierr);
ierr = PCReset_MG(pc);CHKERRQ(ierr);
}
}
/* setup special features of PCML */
/*--------------------------------*/
/* covert A to Aloc to be used by ML at fine grid */
pc_ml->size = size;
ierr = PetscObjectTypeCompare((PetscObject) A, MATSEQAIJ, &isSeq);CHKERRQ(ierr);
ierr = PetscObjectTypeCompare((PetscObject) A, MATMPIAIJ, &isMPI);CHKERRQ(ierr);
if (isMPI){
ierr = MatConvert_MPIAIJ_ML(A,PETSC_NULL,MAT_INITIAL_MATRIX,&Aloc);CHKERRQ(ierr);
} else if (isSeq) {
Aloc = A;
ierr = PetscObjectReference((PetscObject)Aloc);CHKERRQ(ierr);
} else SETERRQ1(((PetscObject)pc)->comm,PETSC_ERR_ARG_WRONG, "Matrix type '%s' cannot be used with ML. ML can only handle AIJ matrices.",((PetscObject)A)->type_name);
/* create and initialize struct 'PetscMLdata' */
ierr = PetscNewLog(pc,FineGridCtx,&PetscMLdata);CHKERRQ(ierr);
pc_ml->PetscMLdata = PetscMLdata;
ierr = PetscMalloc((Aloc->cmap->n+1)*sizeof(PetscScalar),&PetscMLdata->pwork);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&PetscMLdata->x);CHKERRQ(ierr);
ierr = VecSetSizes(PetscMLdata->x,Aloc->cmap->n,Aloc->cmap->n);CHKERRQ(ierr);
ierr = VecSetType(PetscMLdata->x,VECSEQ);CHKERRQ(ierr);
ierr = VecCreate(PETSC_COMM_SELF,&PetscMLdata->y);CHKERRQ(ierr);
ierr = VecSetSizes(PetscMLdata->y,A->rmap->n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetType(PetscMLdata->y,VECSEQ);CHKERRQ(ierr);
PetscMLdata->A = A;
PetscMLdata->Aloc = Aloc;
/* create ML discretization matrix at fine grid */
/* ML requires input of fine-grid matrix. It determines nlevels. */
ierr = MatGetSize(Aloc,&m,&nlocal_allcols);CHKERRQ(ierr);
ierr = MatGetBlockSize(A,&bs);CHKERRQ(ierr);
ML_Create(&ml_object,pc_ml->MaxNlevels);
ML_Comm_Set_UsrComm(ml_object->comm,((PetscObject)A)->comm);
pc_ml->ml_object = ml_object;
ML_Init_Amatrix(ml_object,0,m,m,PetscMLdata);
ML_Set_Amatrix_Getrow(ml_object,0,PetscML_getrow,PetscML_comm,nlocal_allcols);
ML_Set_Amatrix_Matvec(ml_object,0,PetscML_matvec);
ML_Set_Symmetrize(ml_object,pc_ml->Symmetrize ? ML_YES : ML_NO);
/* aggregation */
ML_Aggregate_Create(&agg_object);
pc_ml->agg_object = agg_object;
{
MatNullSpace mnull;
ierr = MatGetNearNullSpace(A,&mnull);CHKERRQ(ierr);
if (pc_ml->nulltype == PCML_NULLSPACE_AUTO) {
if (mnull) pc_ml->nulltype = PCML_NULLSPACE_USER;
else if (bs > 1) pc_ml->nulltype = PCML_NULLSPACE_BLOCK;
else pc_ml->nulltype = PCML_NULLSPACE_SCALAR;
}
switch (pc_ml->nulltype) {
case PCML_NULLSPACE_USER: {
PetscScalar *nullvec;
const PetscScalar *v;
PetscBool has_const;
PetscInt i,j,mlocal,nvec,M;
const Vec *vecs;
if (!mnull) SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_USER,"Must provide explicit null space using MatSetNearNullSpace() to use user-specified null space");
ierr = MatGetSize(A,&M,PETSC_NULL);CHKERRQ(ierr);
ierr = MatGetLocalSize(Aloc,&mlocal,PETSC_NULL);CHKERRQ(ierr);
ierr = MatNullSpaceGetVecs(mnull,&has_const,&nvec,&vecs);CHKERRQ(ierr);
ierr = PetscMalloc((nvec+!!has_const)*mlocal*sizeof *nullvec,&nullvec);CHKERRQ(ierr);
if (has_const) for (i=0; i<mlocal; i++) nullvec[i] = 1.0/M;
for (i=0; i<nvec; i++) {
ierr = VecGetArrayRead(vecs[i],&v);CHKERRQ(ierr);
for (j=0; j<mlocal; j++) nullvec[(i+!!has_const)*mlocal + j] = v[j];
ierr = VecRestoreArrayRead(vecs[i],&v);CHKERRQ(ierr);
}
ierr = ML_Aggregate_Set_NullSpace(agg_object,bs,nvec+!!has_const,nullvec,mlocal);CHKERRQ(ierr);
ierr = PetscFree(nullvec);CHKERRQ(ierr);
} break;
case PCML_NULLSPACE_BLOCK:
ierr = ML_Aggregate_Set_NullSpace(agg_object,bs,bs,0,0);CHKERRQ(ierr);
break;
case PCML_NULLSPACE_SCALAR:
break;
default: SETERRQ(((PetscObject)pc)->comm,PETSC_ERR_SUP,"Unknown null space type");
}
}
ML_Aggregate_Set_MaxCoarseSize(agg_object,pc_ml->MaxCoarseSize);
/* set options */
switch (pc_ml->CoarsenScheme) {
case 1:
ML_Aggregate_Set_CoarsenScheme_Coupled(agg_object);break;
case 2:
ML_Aggregate_Set_CoarsenScheme_MIS(agg_object);break;
case 3:
ML_Aggregate_Set_CoarsenScheme_METIS(agg_object);break;
}
ML_Aggregate_Set_Threshold(agg_object,pc_ml->Threshold);
ML_Aggregate_Set_DampingFactor(agg_object,pc_ml->DampingFactor);
if (pc_ml->SpectralNormScheme_Anorm){
ML_Set_SpectralNormScheme_Anorm(ml_object);
}
agg_object->keep_agg_information = (int)pc_ml->KeepAggInfo;
agg_object->keep_P_tentative = (int)pc_ml->Reusable;
agg_object->block_scaled_SA = (int)pc_ml->BlockScaling;
agg_object->minimizing_energy = (int)pc_ml->EnergyMinimization;
agg_object->minimizing_energy_droptol = (double)pc_ml->EnergyMinimizationDropTol;
agg_object->cheap_minimizing_energy = (int)pc_ml->EnergyMinimizationCheap;
if (pc_ml->OldHierarchy) {
Nlevels = ML_Gen_MGHierarchy_UsingAggregation(ml_object,0,ML_INCREASING,agg_object);
} else {
Nlevels = ML_Gen_MultiLevelHierarchy_UsingAggregation(ml_object,0,ML_INCREASING,agg_object);
}
if (Nlevels<=0) SETERRQ1(((PetscObject)pc)->comm,PETSC_ERR_ARG_OUTOFRANGE,"Nlevels %d must > 0",Nlevels);
pc_ml->Nlevels = Nlevels;
fine_level = Nlevels - 1;
ierr = PCMGSetLevels(pc,Nlevels,PETSC_NULL);CHKERRQ(ierr);
/* set default smoothers */
for (level=1; level<=fine_level; level++){
if (size == 1){
ierr = PCMGGetSmoother(pc,level,&smoother);CHKERRQ(ierr);
ierr = KSPSetType(smoother,KSPRICHARDSON);CHKERRQ(ierr);
ierr = KSPGetPC(smoother,&subpc);CHKERRQ(ierr);
ierr = PCSetType(subpc,PCSOR);CHKERRQ(ierr);
} else {
ierr = PCMGGetSmoother(pc,level,&smoother);CHKERRQ(ierr);
ierr = KSPSetType(smoother,KSPRICHARDSON);CHKERRQ(ierr);
ierr = KSPGetPC(smoother,&subpc);CHKERRQ(ierr);
ierr = PCSetType(subpc,PCSOR);CHKERRQ(ierr);
}
}
ierr = PetscObjectOptionsBegin((PetscObject)pc);CHKERRQ(ierr);
ierr = PCSetFromOptions_MG(pc);CHKERRQ(ierr); /* should be called in PCSetFromOptions_ML(), but cannot be called prior to PCMGSetLevels() */
ierr = PetscOptionsEnd();CHKERRQ(ierr);
ierr = PetscMalloc(Nlevels*sizeof(GridCtx),&gridctx);CHKERRQ(ierr);
pc_ml->gridctx = gridctx;
/* wrap ML matrices by PETSc shell matrices at coarsened grids.
Level 0 is the finest grid for ML, but coarsest for PETSc! */
gridctx[fine_level].A = A;
level = fine_level - 1;
if (size == 1){ /* convert ML P, R and A into seqaij format */
for (mllevel=1; mllevel<Nlevels; mllevel++){
mlmat = &(ml_object->Pmat[mllevel]);
ierr = MatWrapML_SeqAIJ(mlmat,MAT_INITIAL_MATRIX,&gridctx[level].P);CHKERRQ(ierr);
mlmat = &(ml_object->Rmat[mllevel-1]);
ierr = MatWrapML_SeqAIJ(mlmat,MAT_INITIAL_MATRIX,&gridctx[level].R);CHKERRQ(ierr);
mlmat = &(ml_object->Amat[mllevel]);
ierr = MatWrapML_SeqAIJ(mlmat,MAT_INITIAL_MATRIX,&gridctx[level].A);CHKERRQ(ierr);
level--;
}
} else { /* convert ML P and R into shell format, ML A into mpiaij format */
for (mllevel=1; mllevel<Nlevels; mllevel++){
mlmat = &(ml_object->Pmat[mllevel]);
ierr = MatWrapML_SHELL(mlmat,MAT_INITIAL_MATRIX,&gridctx[level].P);CHKERRQ(ierr);
mlmat = &(ml_object->Rmat[mllevel-1]);
ierr = MatWrapML_SHELL(mlmat,MAT_INITIAL_MATRIX,&gridctx[level].R);CHKERRQ(ierr);
mlmat = &(ml_object->Amat[mllevel]);
ierr = MatWrapML_MPIAIJ(mlmat,MAT_INITIAL_MATRIX,&gridctx[level].A);CHKERRQ(ierr);
level--;
}
}
/* create vectors and ksp at all levels */
for (level=0; level<fine_level; level++){
level1 = level + 1;
ierr = VecCreate(((PetscObject)gridctx[level].A)->comm,&gridctx[level].x);CHKERRQ(ierr);
ierr = VecSetSizes(gridctx[level].x,gridctx[level].A->cmap->n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetType(gridctx[level].x,VECMPI);CHKERRQ(ierr);
ierr = PCMGSetX(pc,level,gridctx[level].x);CHKERRQ(ierr);
ierr = VecCreate(((PetscObject)gridctx[level].A)->comm,&gridctx[level].b);CHKERRQ(ierr);
ierr = VecSetSizes(gridctx[level].b,gridctx[level].A->rmap->n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetType(gridctx[level].b,VECMPI);CHKERRQ(ierr);
ierr = PCMGSetRhs(pc,level,gridctx[level].b);CHKERRQ(ierr);
ierr = VecCreate(((PetscObject)gridctx[level1].A)->comm,&gridctx[level1].r);CHKERRQ(ierr);
ierr = VecSetSizes(gridctx[level1].r,gridctx[level1].A->rmap->n,PETSC_DECIDE);CHKERRQ(ierr);
ierr = VecSetType(gridctx[level1].r,VECMPI);CHKERRQ(ierr);
ierr = PCMGSetR(pc,level1,gridctx[level1].r);CHKERRQ(ierr);
if (level == 0){
ierr = PCMGGetCoarseSolve(pc,&gridctx[level].ksp);CHKERRQ(ierr);
} else {
ierr = PCMGGetSmoother(pc,level,&gridctx[level].ksp);CHKERRQ(ierr);
}
}
ierr = PCMGGetSmoother(pc,fine_level,&gridctx[fine_level].ksp);CHKERRQ(ierr);
/* create coarse level and the interpolation between the levels */
for (level=0; level<fine_level; level++){
level1 = level + 1;
ierr = PCMGSetInterpolation(pc,level1,gridctx[level].P);CHKERRQ(ierr);
ierr = PCMGSetRestriction(pc,level1,gridctx[level].R);CHKERRQ(ierr);
if (level > 0){
ierr = PCMGSetResidual(pc,level,PCMGDefaultResidual,gridctx[level].A);CHKERRQ(ierr);
}
ierr = KSPSetOperators(gridctx[level].ksp,gridctx[level].A,gridctx[level].A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
}
ierr = PCMGSetResidual(pc,fine_level,PCMGDefaultResidual,gridctx[fine_level].A);CHKERRQ(ierr);
ierr = KSPSetOperators(gridctx[fine_level].ksp,gridctx[level].A,gridctx[fine_level].A,DIFFERENT_NONZERO_PATTERN);CHKERRQ(ierr);
/* setupcalled is set to 0 so that MG is setup from scratch */
pc->setupcalled = 0;
ierr = PCSetUp_MG(pc);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc, char *argv[]){
ML *ml_object;
int i, N_grids = 3, N_levels;
double sol[5], rhs[5];
ML_Aggregate *agg_object;
int proc, nlocal, nlocal_allcolumns;
MPI_Init(&argc,&argv);
ML_Set_PrintLevel(15);
for (i = 0; i < 5; i++) sol[i] = 0.;
for (i = 0; i < 5; i++) rhs[i] = 2.;
ML_Create (&ml_object, N_grids);
proc = ml_object->comm->ML_mypid;
if (ml_object->comm->ML_nprocs != 2) {
if (proc == 0) printf("Must be run on two processors\n");
ML_Destroy(&ml_object);
MPI_Finalize();
exit(1);
}
if (proc == 0) {nlocal = 2; nlocal_allcolumns = 4;}
else if (proc == 1){nlocal = 3; nlocal_allcolumns = 5;}
else {nlocal = 0; nlocal_allcolumns = 0;}
ML_Init_Amatrix (ml_object, 0, nlocal, nlocal, &proc);
ML_Set_Amatrix_Getrow(ml_object, 0, Poisson_getrow, Poisson_comm,
nlocal_allcolumns);
ML_Set_Amatrix_Matvec(ml_object, 0, Poisson_matvec);
ML_Aggregate_Create(&agg_object);
ML_Aggregate_Set_MaxCoarseSize(agg_object,1);
N_levels = ML_Gen_MGHierarchy_UsingAggregation(ml_object, 0,
ML_INCREASING, agg_object);
ML_Gen_Smoother_Jacobi(ml_object, ML_ALL_LEVELS, ML_PRESMOOTHER, 1, ML_DEFAULT);
ML_Gen_Solver (ml_object, ML_MGV, 0, N_levels-1);
ML_Iterate(ml_object, sol, rhs);
if (proc == 0) {
printf("sol(0) = %e\n",sol[1]);
fflush(stdout);
}
ML_Comm_GsumInt(ml_object->comm,1); /* just used for synchronization */
if (proc == 1) {
printf("sol(1) = %e\n",sol[0]);
printf("sol(2) = %e\n",sol[1]);
printf("sol(3) = %e\n",sol[2]);
fflush(stdout);
}
ML_Comm_GsumInt(ml_object->comm,1); /* just used for synchronization */
if (proc == 0) {
printf("sol(4) = %e\n",sol[0]);
fflush(stdout);
}
ML_Aggregate_Destroy(&agg_object);
ML_Destroy(&ml_object);
MPI_Finalize();
return 0;
}
HYPRE_Int HYPRE_ParCSRMLSetup( HYPRE_Solver solver, HYPRE_ParCSRMatrix A,
HYPRE_ParVector b, HYPRE_ParVector x )
{
HYPRE_Int i, my_id, nprocs, coarsest_level, level, sweeps, nlevels;
HYPRE_Int *row_partition, localEqns, length;
HYPRE_Int Nblocks, *blockList;
double wght;
MH_Context *context;
MH_Matrix *mh_mat;
/* -------------------------------------------------------- */
/* fetch the ML pointer */
/* -------------------------------------------------------- */
MH_Link *link = (MH_Link *) solver;
ML *ml = link->ml_ptr;
nlevels = link->nlevels;
/* -------------------------------------------------------- */
/* set up the parallel environment */
/* -------------------------------------------------------- */
hypre_MPI_Comm_rank(link->comm, &my_id);
hypre_MPI_Comm_size(link->comm, &nprocs);
/* -------------------------------------------------------- */
/* fetch the matrix row partition information and put it */
/* into the matrix data object (for matvec and getrow) */
/* -------------------------------------------------------- */
HYPRE_ParCSRMatrixGetRowPartitioning( A, &row_partition );
localEqns = row_partition[my_id+1] - row_partition[my_id];
context = (MH_Context *) malloc(sizeof(MH_Context));
link->contxt = context;
context->comm = link->comm;
context->globalEqns = row_partition[nprocs];
context->partition = (HYPRE_Int *) malloc(sizeof(HYPRE_Int)*(nprocs+1));
for (i=0; i<=nprocs; i++) context->partition[i] = row_partition[i];
hypre_TFree( row_partition );
mh_mat = ( MH_Matrix * ) malloc( sizeof( MH_Matrix) );
context->Amat = mh_mat;
HYPRE_ParCSRMLConstructMHMatrix(A,mh_mat,link->comm,
context->partition,context);
/* -------------------------------------------------------- */
/* set up the ML communicator information */
/* -------------------------------------------------------- */
ML_Set_Comm_Communicator(ml, link->comm);
ML_Set_Comm_MyRank(ml, my_id);
ML_Set_Comm_Nprocs(ml, nprocs);
ML_Set_Comm_Send(ml, MH_Send);
ML_Set_Comm_Recv(ml, MH_Irecv);
ML_Set_Comm_Wait(ml, MH_Wait);
/* -------------------------------------------------------- */
/* set up the ML matrix information */
/* -------------------------------------------------------- */
ML_Init_Amatrix(ml, nlevels-1, localEqns, localEqns, (void *) context);
ML_Set_Amatrix_Matvec(ml, nlevels-1, MH_MatVec);
length = localEqns;
for (i=0; i<mh_mat->recvProcCnt; i++ ) length += mh_mat->recvLeng[i];
ML_Set_Amatrix_Getrow(ml, nlevels-1, MH_GetRow, MH_ExchBdry, length);
/* -------------------------------------------------------- */
/* create an aggregate context */
/* -------------------------------------------------------- */
ML_Aggregate_Create(&(link->ml_ag));
link->ml_ag->max_levels = link->nlevels;
ML_Aggregate_Set_Threshold( link->ml_ag, link->ag_threshold );
/* -------------------------------------------------------- */
/* perform aggregation */
/* -------------------------------------------------------- */
coarsest_level = ML_Gen_MGHierarchy_UsingAggregation(ml, nlevels-1,
ML_DECREASING, link->ml_ag);
if ( my_id == 0 )
hypre_printf("ML : number of levels = %d\n", coarsest_level);
coarsest_level = nlevels - coarsest_level;
/* -------------------------------------------------------- */
/* set up smoother and coarse solver */
/* -------------------------------------------------------- */
for (level = nlevels-1; level > coarsest_level; level--)
{
sweeps = link->pre_sweeps;
wght = link->jacobi_wt;
switch ( link->pre )
{
case 0 :
ML_Gen_SmootherJacobi(ml, level, ML_PRESMOOTHER, sweeps, wght);
break;
case 1 :
ML_Gen_SmootherGaussSeidel(ml, level, ML_PRESMOOTHER, sweeps);
break;
case 2 :
ML_Gen_SmootherSymGaussSeidel(ml,level,ML_PRESMOOTHER,sweeps,1.0);
break;
case 3 :
Nblocks = ML_Aggregate_Get_AggrCount( link->ml_ag, level );
ML_Aggregate_Get_AggrMap( link->ml_ag, level, &blockList );
ML_Gen_SmootherVBlockGaussSeidel(ml,level,ML_PRESMOOTHER,
sweeps, Nblocks, blockList);
break;
case 4 :
Nblocks = ML_Aggregate_Get_AggrCount( link->ml_ag, level );
ML_Aggregate_Get_AggrMap( link->ml_ag, level, &blockList );
ML_Gen_SmootherVBlockJacobi(ml,level,ML_PRESMOOTHER,
sweeps, wght, Nblocks, blockList);
break;
}
sweeps = link->post_sweeps;
switch ( link->post )
{
case 0 :
ML_Gen_SmootherJacobi(ml, level, ML_POSTSMOOTHER, sweeps, wght);
break;
case 1 :
ML_Gen_SmootherGaussSeidel(ml, level, ML_POSTSMOOTHER, sweeps);
break;
case 2 :
ML_Gen_SmootherSymGaussSeidel(ml,level,ML_POSTSMOOTHER,sweeps,1.0);
break;
case 3 :
Nblocks = ML_Aggregate_Get_AggrCount( link->ml_ag, level );
ML_Aggregate_Get_AggrMap( link->ml_ag, level, &blockList );
ML_Gen_SmootherVBlockGaussSeidel(ml,level,ML_POSTSMOOTHER,
sweeps, Nblocks, blockList);
break;
case 4 :
Nblocks = ML_Aggregate_Get_AggrCount( link->ml_ag, level );
ML_Aggregate_Get_AggrMap( link->ml_ag, level, &blockList );
ML_Gen_SmootherVBlockJacobi(ml,level,ML_POSTSMOOTHER,
sweeps, wght, Nblocks, blockList);
break;
}
}
ML_Gen_CoarseSolverSuperLU(ml, coarsest_level);
//ML_Gen_SmootherGaussSeidel(ml, coarsest_level, ML_PRESMOOTHER, 100);
ML_Gen_Solver(ml, ML_MGV, nlevels-1, coarsest_level);
return 0;
}
int main(int argc, char *argv[])
{
int num_PDE_eqns=1, N_levels=3, nsmooth=2;
int leng, level, N_grid_pts, coarsest_level;
int leng1,leng2;
/* See Aztec User's Guide for more information on the */
/* variables that follow. */
int proc_config[AZ_PROC_SIZE], options[AZ_OPTIONS_SIZE];
double params[AZ_PARAMS_SIZE], status[AZ_STATUS_SIZE];
/* data structure for matrix corresponding to the fine grid */
double *val = NULL, *xxx, *rhs, solve_time, setup_time, start_time;
AZ_MATRIX *Amat;
AZ_PRECOND *Pmat = NULL;
ML *ml;
FILE *fp;
int i, j, Nrigid, *garbage, nblocks=0, *blocks = NULL, *block_pde=NULL;
struct AZ_SCALING *scaling;
ML_Aggregate *ag;
double *mode, *rigid=NULL, alpha;
char filename[80];
int one = 1;
int proc,nprocs;
char pathfilename[100];
#ifdef ML_MPI
MPI_Init(&argc,&argv);
/* get number of processors and the name of this processor */
AZ_set_proc_config(proc_config, MPI_COMM_WORLD);
proc = proc_config[AZ_node];
nprocs = proc_config[AZ_N_procs];
#else
AZ_set_proc_config(proc_config, AZ_NOT_MPI);
proc = 0;
nprocs = 1;
#endif
if (proc_config[AZ_node] == 0) {
sprintf(pathfilename,"%s/inputfile",argv[1]);
ML_Reader_ReadInput(pathfilename, &context);
}
else context = (struct reader_context *) ML_allocate(sizeof(struct reader_context));
AZ_broadcast((char *) context, sizeof(struct reader_context), proc_config,
AZ_PACK);
AZ_broadcast((char *) NULL , 0 , proc_config, AZ_SEND);
N_levels = context->N_levels;
printf("N_levels %d\n",N_levels);
nsmooth = context->nsmooth;
num_PDE_eqns = context->N_dofPerNode;
printf("num_PDE_eqns %d\n",num_PDE_eqns);
ML_Set_PrintLevel(context->output_level);
/* read in the number of matrix equations */
leng = 0;
if (proc_config[AZ_node] == 0) {
sprintf(pathfilename,"%s/data_matrix.txt",argv[1]);
fp=fopen(pathfilename,"r");
if (fp==NULL) {
printf("**ERR** couldn't open file data_matrix.txt\n");
exit(1);
}
fscanf(fp,"%d",&leng);
fclose(fp);
}
leng = AZ_gsum_int(leng, proc_config);
N_grid_pts=leng/num_PDE_eqns;
/* initialize the list of global indices. NOTE: the list of global */
/* indices must be in ascending order so that subsequent calls to */
/* AZ_find_index() will function properly. */
#if 0
if (proc_config[AZ_N_procs] == 1) i = AZ_linear;
else i = AZ_file;
#endif
i = AZ_linear;
/* cannot use AZ_input_update for variable blocks (forgot why, but debugged through it)*/
/* make a linear distribution of the matrix */
/* if the linear distribution does not align with the blocks, */
/* this is corrected in ML_AZ_Reader_ReadVariableBlocks */
leng1 = leng/nprocs;
leng2 = leng-leng1*nprocs;
if (proc >= leng2)
{
leng2 += (proc*leng1);
}
else
{
leng1++;
leng2 = proc*leng1;
}
N_update = leng1;
update = (int*)AZ_allocate((N_update+1)*sizeof(int));
if (update==NULL)
{
(void) fprintf (stderr, "Not enough space to allocate 'update'\n");
fflush(stderr); exit(EXIT_FAILURE);
}
for (i=0; i<N_update; i++) update[i] = i+leng2;
#if 0 /* debug */
printf("proc %d N_update %d\n",proc_config[AZ_node],N_update);
fflush(stdout);
#endif
sprintf(pathfilename,"%s/data_vblocks.txt",argv[1]);
ML_AZ_Reader_ReadVariableBlocks(pathfilename,&nblocks,&blocks,&block_pde,
&N_update,&update,proc_config);
#if 0 /* debug */
printf("proc %d N_update %d\n",proc_config[AZ_node],N_update);
fflush(stdout);
#endif
sprintf(pathfilename,"%s/data_matrix.txt",argv[1]);
AZ_input_msr_matrix(pathfilename,update, &val, &bindx, N_update, proc_config);
/* This code is to fix things up so that we are sure we have */
/* all blocks (including the ghost nodes) the same size. */
/* not sure, whether this is a good idea with variable blocks */
/* the examples inpufiles (see top of this file) don't need it */
/* anyway */
/*
AZ_block_MSR(&bindx, &val, N_update, num_PDE_eqns, update);
*/
AZ_transform_norowreordering(proc_config, &external, bindx, val, update, &update_index,
&extern_index, &data_org, N_update, 0, 0, 0, &cpntr,
AZ_MSR_MATRIX);
Amat = AZ_matrix_create( leng );
AZ_set_MSR(Amat, bindx, val, data_org, 0, NULL, AZ_LOCAL);
Amat->matrix_type = data_org[AZ_matrix_type];
data_org[AZ_N_rows] = data_org[AZ_N_internal] + data_org[AZ_N_border];
start_time = AZ_second();
options[AZ_scaling] = AZ_none;
ML_Create(&ml, N_levels);
/* set up discretization matrix and matrix vector function */
AZ_ML_Set_Amat(ml, 0, N_update, N_update, Amat, proc_config);
ML_Set_ResidualOutputFrequency(ml, context->output);
ML_Set_Tolerance(ml, context->tol);
ML_Aggregate_Create( &ag );
if (ML_strcmp(context->agg_coarsen_scheme,"Mis") == 0) {
ML_Aggregate_Set_CoarsenScheme_MIS(ag);
}
else if (ML_strcmp(context->agg_coarsen_scheme,"Uncoupled") == 0) {
ML_Aggregate_Set_CoarsenScheme_Uncoupled(ag);
}
else if (ML_strcmp(context->agg_coarsen_scheme,"Coupled") == 0) {
ML_Aggregate_Set_CoarsenScheme_Coupled(ag);
}
else if (ML_strcmp(context->agg_coarsen_scheme,"Metis") == 0) {
ML_Aggregate_Set_CoarsenScheme_METIS(ag);
for (i=0; i<N_levels; i++)
ML_Aggregate_Set_NodesPerAggr(ml,ag,i,9);
}
else if (ML_strcmp(context->agg_coarsen_scheme,"VBMetis") == 0) {
/* when no blocks read, use standard metis assuming constant block sizes */
if (!blocks)
ML_Aggregate_Set_CoarsenScheme_METIS(ag);
else {
ML_Aggregate_Set_CoarsenScheme_VBMETIS(ag);
ML_Aggregate_Set_Vblocks_CoarsenScheme_VBMETIS(ag,0,N_levels,nblocks,
blocks,block_pde,N_update);
}
for (i=0; i<N_levels; i++)
ML_Aggregate_Set_NodesPerAggr(ml,ag,i,9);
}
else {
printf("**ERR** ML: Unknown aggregation scheme %s\n",context->agg_coarsen_scheme);
exit(-1);
}
ML_Aggregate_Set_DampingFactor(ag, context->agg_damping);
ML_Aggregate_Set_MaxCoarseSize( ag, context->maxcoarsesize);
ML_Aggregate_Set_Threshold(ag, context->agg_thresh);
if (ML_strcmp(context->agg_spectral_norm,"Calc") == 0) {
ML_Set_SpectralNormScheme_Calc(ml);
}
else if (ML_strcmp(context->agg_spectral_norm,"Anorm") == 0) {
ML_Set_SpectralNormScheme_Anorm(ml);
}
else {
printf("**WRN** ML: Unknown spectral norm scheme %s\n",context->agg_spectral_norm);
}
/* read in the rigid body modes */
Nrigid = 0;
if (proc_config[AZ_node] == 0) {
sprintf(filename,"data_nullsp%d.txt",Nrigid);
sprintf(pathfilename,"%s/%s",argv[1],filename);
while( (fp = fopen(pathfilename,"r")) != NULL) {
fclose(fp);
Nrigid++;
sprintf(filename,"data_nullsp%d.txt",Nrigid);
sprintf(pathfilename,"%s/%s",argv[1],filename);
}
}
Nrigid = AZ_gsum_int(Nrigid,proc_config);
if (Nrigid != 0) {
rigid = (double *) ML_allocate( sizeof(double)*Nrigid*(N_update+1) );
if (rigid == NULL) {
printf("Error: Not enough space for rigid body modes\n");
}
}
/* Set rhs */
sprintf(pathfilename,"%s/data_rhs.txt",argv[1]);
fp = fopen(pathfilename,"r");
if (fp == NULL) {
rhs=(double *)ML_allocate(leng*sizeof(double));
if (proc_config[AZ_node] == 0) printf("taking linear vector for rhs\n");
for (i = 0; i < N_update; i++) rhs[i] = (double) update[i];
}
else {
fclose(fp);
if (proc_config[AZ_node] == 0) printf("reading rhs from a file\n");
AZ_input_msr_matrix(pathfilename, update, &rhs, &garbage, N_update,
proc_config);
}
AZ_reorder_vec(rhs, data_org, update_index, NULL);
for (i = 0; i < Nrigid; i++) {
sprintf(filename,"data_nullsp%d.txt",i);
sprintf(pathfilename,"%s/%s",argv[1],filename);
AZ_input_msr_matrix(pathfilename, update, &mode, &garbage, N_update,
proc_config);
AZ_reorder_vec(mode, data_org, update_index, NULL);
#if 0 /* test the given rigid body mode, output-vector should be ~0 */
Amat->matvec(mode, rigid, Amat, proc_config);
for (j = 0; j < N_update; j++) printf("this is %d %e\n",j,rigid[j]);
#endif
for (j = 0; j < i; j++) {
alpha = -AZ_gdot(N_update, mode, &(rigid[j*N_update]), proc_config)/
AZ_gdot(N_update, &(rigid[j*N_update]), &(rigid[j*N_update]),
proc_config);
DAXPY_F77(&N_update, &alpha, &(rigid[j*N_update]), &one, mode, &one);
}
/* rhs orthogonalization */
alpha = -AZ_gdot(N_update, mode, rhs, proc_config)/
AZ_gdot(N_update, mode, mode, proc_config);
DAXPY_F77(&N_update, &alpha, mode, &one, rhs, &one);
for (j = 0; j < N_update; j++) rigid[i*N_update+j] = mode[j];
free(mode);
free(garbage);
}
for (j = 0; j < Nrigid; j++) {
alpha = -AZ_gdot(N_update, rhs, &(rigid[j*N_update]), proc_config)/
AZ_gdot(N_update, &(rigid[j*N_update]), &(rigid[j*N_update]),
proc_config);
DAXPY_F77(&N_update, &alpha, &(rigid[j*N_update]), &one, rhs, &one);
}
#if 0 /* for testing the default nullsp */
ML_Aggregate_Set_NullSpace(ag, num_PDE_eqns, 6, NULL, N_update);
#else
if (Nrigid != 0) {
ML_Aggregate_Set_NullSpace(ag, num_PDE_eqns, Nrigid, rigid, N_update);
}
#endif
if (rigid) ML_free(rigid);
ag->keep_agg_information = 1;
coarsest_level = ML_Gen_MGHierarchy_UsingAggregation(ml, 0,
ML_INCREASING, ag);
coarsest_level--;
if ( proc_config[AZ_node] == 0 )
printf("Coarse level = %d \n", coarsest_level);
#if 0
/* set up smoothers */
if (!blocks)
blocks = (int *) ML_allocate(sizeof(int)*N_update);
#endif
for (level = 0; level < coarsest_level; level++) {
num_PDE_eqns = ml->Amat[level].num_PDEs;
/* Sparse approximate inverse smoother that acutally does both */
/* pre and post smoothing. */
if (ML_strcmp(context->smoother,"Parasails") == 0) {
ML_Gen_Smoother_ParaSails(ml , level, ML_PRESMOOTHER, nsmooth,
parasails_sym, parasails_thresh,
parasails_nlevels, parasails_filter,
(int) parasails_loadbal, parasails_factorized);
}
/* This is the symmetric Gauss-Seidel smoothing that we usually use. */
/* In parallel, it is not a true Gauss-Seidel in that each processor */
/* does a Gauss-Seidel on its local submatrix independent of the */
/* other processors. */
else if (ML_strcmp(context->smoother,"GaussSeidel") == 0) {
ML_Gen_Smoother_GaussSeidel(ml , level, ML_BOTH, nsmooth,1.);
}
else if (ML_strcmp(context->smoother,"SymGaussSeidel") == 0) {
ML_Gen_Smoother_SymGaussSeidel(ml , level, ML_BOTH, nsmooth,1.);
}
else if (ML_strcmp(context->smoother,"Poly") == 0) {
ML_Gen_Smoother_Cheby(ml, level, ML_BOTH, 30., nsmooth);
}
else if (ML_strcmp(context->smoother,"BlockGaussSeidel") == 0) {
ML_Gen_Smoother_BlockGaussSeidel(ml , level, ML_BOTH, nsmooth,1.,
num_PDE_eqns);
}
else if (ML_strcmp(context->smoother,"VBSymGaussSeidel") == 0) {
if (blocks) ML_free(blocks);
if (block_pde) ML_free(block_pde);
blocks = NULL;
block_pde = NULL;
nblocks = 0;
ML_Aggregate_Get_Vblocks_CoarsenScheme_VBMETIS(ag,level,N_levels,&nblocks,
&blocks,&block_pde);
if (blocks==NULL) ML_Gen_Blocks_Aggregates(ag, level, &nblocks, &blocks);
ML_Gen_Smoother_VBlockSymGaussSeidel(ml , level, ML_BOTH, nsmooth,1.,
nblocks, blocks);
}
/* This is a true Gauss Seidel in parallel. This seems to work for */
/* elasticity problems. However, I don't believe that this is very */
/* efficient in parallel. */
/*
nblocks = ml->Amat[level].invec_leng;
for (i =0; i < nblocks; i++) blocks[i] = i;
ML_Gen_Smoother_VBlockSymGaussSeidelSequential(ml , level, ML_PRESMOOTHER,
nsmooth, 1., nblocks, blocks);
ML_Gen_Smoother_VBlockSymGaussSeidelSequential(ml, level, ML_POSTSMOOTHER,
nsmooth, 1., nblocks, blocks);
*/
/* Jacobi Smoothing */
else if (ML_strcmp(context->smoother,"Jacobi") == 0) {
ML_Gen_Smoother_Jacobi(ml , level, ML_PRESMOOTHER, nsmooth,.4);
ML_Gen_Smoother_Jacobi(ml , level, ML_POSTSMOOTHER, nsmooth,.4);
}
/* This does a block Gauss-Seidel (not true GS in parallel) */
/* where each processor has 'nblocks' blocks. */
/* */
else if (ML_strcmp(context->smoother,"Metis") == 0) {
if (blocks) ML_free(blocks);
if (block_pde) ML_free(block_pde);
nblocks = 250;
ML_Gen_Blocks_Metis(ml, level, &nblocks, &blocks);
ML_Gen_Smoother_VBlockSymGaussSeidel(ml , level, ML_BOTH, nsmooth,1.,
nblocks, blocks);
}
else {
printf("unknown smoother %s\n",context->smoother);
exit(1);
}
}
/* set coarse level solver */
nsmooth = context->coarse_its;
/* Sparse approximate inverse smoother that acutally does both */
/* pre and post smoothing. */
if (ML_strcmp(context->coarse_solve,"Parasails") == 0) {
ML_Gen_Smoother_ParaSails(ml , coarsest_level, ML_PRESMOOTHER, nsmooth,
parasails_sym, parasails_thresh,
parasails_nlevels, parasails_filter,
(int) parasails_loadbal, parasails_factorized);
}
else if (ML_strcmp(context->coarse_solve,"GaussSeidel") == 0) {
ML_Gen_Smoother_GaussSeidel(ml , coarsest_level, ML_BOTH, nsmooth,1.);
}
else if (ML_strcmp(context->coarse_solve,"Poly") == 0) {
ML_Gen_Smoother_Cheby(ml, coarsest_level, ML_BOTH, 30., nsmooth);
}
else if (ML_strcmp(context->coarse_solve,"SymGaussSeidel") == 0) {
ML_Gen_Smoother_SymGaussSeidel(ml , coarsest_level, ML_BOTH, nsmooth,1.);
}
else if (ML_strcmp(context->coarse_solve,"BlockGaussSeidel") == 0) {
ML_Gen_Smoother_BlockGaussSeidel(ml, coarsest_level, ML_BOTH, nsmooth,1.,
num_PDE_eqns);
}
else if (ML_strcmp(context->coarse_solve,"Aggregate") == 0) {
if (blocks) ML_free(blocks);
if (block_pde) ML_free(block_pde);
ML_Gen_Blocks_Aggregates(ag, coarsest_level, &nblocks, &blocks);
ML_Gen_Smoother_VBlockSymGaussSeidel(ml , coarsest_level, ML_BOTH,
nsmooth,1., nblocks, blocks);
}
else if (ML_strcmp(context->coarse_solve,"Jacobi") == 0) {
ML_Gen_Smoother_Jacobi(ml , coarsest_level, ML_BOTH, nsmooth,.5);
}
else if (ML_strcmp(context->coarse_solve,"Metis") == 0) {
if (blocks) ML_free(blocks);
if (block_pde) ML_free(block_pde);
nblocks = 250;
ML_Gen_Blocks_Metis(ml, coarsest_level, &nblocks, &blocks);
ML_Gen_Smoother_VBlockSymGaussSeidel(ml , coarsest_level, ML_BOTH,
nsmooth,1., nblocks, blocks);
}
else if (ML_strcmp(context->coarse_solve,"SuperLU") == 0) {
ML_Gen_CoarseSolverSuperLU( ml, coarsest_level);
}
else if (ML_strcmp(context->coarse_solve,"Amesos") == 0) {
ML_Gen_Smoother_Amesos(ml,coarsest_level,ML_AMESOS_KLU,-1, 0.0);
}
else {
printf("unknown coarse grid solver %s\n",context->coarse_solve);
exit(1);
}
ML_Gen_Solver(ml, ML_MGV, 0, coarsest_level);
AZ_defaults(options, params);
if (ML_strcmp(context->krylov,"Cg") == 0) {
options[AZ_solver] = AZ_cg;
}
else if (ML_strcmp(context->krylov,"Bicgstab") == 0) {
options[AZ_solver] = AZ_bicgstab;
}
else if (ML_strcmp(context->krylov,"Tfqmr") == 0) {
options[AZ_solver] = AZ_tfqmr;
}
else if (ML_strcmp(context->krylov,"Gmres") == 0) {
options[AZ_solver] = AZ_gmres;
}
else {
printf("unknown krylov method %s\n",context->krylov);
}
if (blocks) ML_free(blocks);
if (block_pde) ML_free(block_pde);
options[AZ_scaling] = AZ_none;
options[AZ_precond] = AZ_user_precond;
options[AZ_conv] = AZ_r0;
options[AZ_output] = 1;
options[AZ_max_iter] = context->max_outer_its;
options[AZ_poly_ord] = 5;
options[AZ_kspace] = 130;
params[AZ_tol] = context->tol;
options[AZ_output] = context->output;
ML_free(context);
AZ_set_ML_preconditioner(&Pmat, Amat, ml, options);
setup_time = AZ_second() - start_time;
xxx = (double *) malloc( leng*sizeof(double));
for (iii = 0; iii < leng; iii++) xxx[iii] = 0.0;
/* Set x */
/*
there is no initguess supplied with these examples for the moment....
*/
fp = fopen("initguessfile","r");
if (fp != NULL) {
fclose(fp);
if (proc_config[AZ_node]== 0) printf("reading initial guess from file\n");
AZ_input_msr_matrix("data_initguess.txt", update, &xxx, &garbage, N_update,
proc_config);
options[AZ_conv] = AZ_expected_values;
}
else if (proc_config[AZ_node]== 0) printf("taking 0 initial guess \n");
AZ_reorder_vec(xxx, data_org, update_index, NULL);
/* if Dirichlet BC ... put the answer in */
for (i = 0; i < data_org[AZ_N_internal]+data_org[AZ_N_border]; i++) {
if ( (val[i] > .99999999) && (val[i] < 1.0000001))
xxx[i] = rhs[i];
}
fp = fopen("AZ_no_multilevel.dat","r");
scaling = AZ_scaling_create();
start_time = AZ_second();
if (fp != NULL) {
fclose(fp);
options[AZ_precond] = AZ_none;
options[AZ_scaling] = AZ_sym_diag;
options[AZ_ignore_scaling] = AZ_TRUE;
options[AZ_keep_info] = 1;
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
/*
options[AZ_pre_calc] = AZ_reuse;
options[AZ_conv] = AZ_expected_values;
if (proc_config[AZ_node] == 0)
printf("\n-------- Second solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
if (proc_config[AZ_node] == 0)
printf("\n-------- Third solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
*/
}
else {
options[AZ_keep_info] = 1;
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
options[AZ_pre_calc] = AZ_reuse;
options[AZ_conv] = AZ_expected_values;
/*
if (proc_config[AZ_node] == 0)
printf("\n-------- Second solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
if (proc_config[AZ_node] == 0)
printf("\n-------- Third solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
*/
}
solve_time = AZ_second() - start_time;
if (proc_config[AZ_node] == 0)
printf("Solve time = %e, MG Setup time = %e\n", solve_time, setup_time);
if (proc_config[AZ_node] == 0)
printf("Printing out a few entries of the solution ...\n");
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 7) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
j = AZ_gsum_int(7, proc_config); /* sync processors */
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 23) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
j = AZ_gsum_int(7, proc_config); /* sync processors */
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 47) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
j = AZ_gsum_int(7, proc_config); /* sync processors */
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 101) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
j = AZ_gsum_int(7, proc_config); /* sync processors */
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 171) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
ML_Aggregate_Destroy(&ag);
ML_Destroy(&ml);
AZ_free((void *) Amat->data_org);
AZ_free((void *) Amat->val);
AZ_free((void *) Amat->bindx);
AZ_free((void *) update);
AZ_free((void *) external);
AZ_free((void *) extern_index);
AZ_free((void *) update_index);
AZ_scaling_destroy(&scaling);
if (Amat != NULL) AZ_matrix_destroy(&Amat);
if (Pmat != NULL) AZ_precond_destroy(&Pmat);
free(xxx);
free(rhs);
#ifdef ML_MPI
MPI_Finalize();
#endif
return 0;
}
int main(int argc, char *argv[])
{
int num_PDE_eqns=3, N_levels=3, nsmooth=1;
int leng, level, N_grid_pts, coarsest_level;
/* See Aztec User's Guide for more information on the */
/* variables that follow. */
int proc_config[AZ_PROC_SIZE], options[AZ_OPTIONS_SIZE];
double params[AZ_PARAMS_SIZE], status[AZ_STATUS_SIZE];
/* data structure for matrix corresponding to the fine grid */
int *data_org = NULL, *update = NULL, *external = NULL;
int *update_index = NULL, *extern_index = NULL;
int *cpntr = NULL;
int *bindx = NULL, N_update, iii;
double *val = NULL;
double *xxx, *rhs;
AZ_MATRIX *Amat;
AZ_PRECOND *Pmat = NULL;
ML *ml;
FILE *fp;
int ch,i,j, Nrigid, *garbage;
struct AZ_SCALING *scaling;
double solve_time, setup_time, start_time, *mode, *rigid;
ML_Aggregate *ag;
int nblocks, *blocks;
char filename[80];
double alpha;
int one = 1;
#ifdef ML_MPI
MPI_Init(&argc,&argv);
/* get number of processors and the name of this processor */
AZ_set_proc_config(proc_config, MPI_COMM_WORLD);
#else
AZ_set_proc_config(proc_config, AZ_NOT_MPI);
#endif
leng = 0;
if (proc_config[AZ_node] == 0) {
#ifdef binary
fp=fopen(".data","rb");
#else
fp=fopen(".data","r");
#endif
if (fp==NULL)
{
printf("couldn't open file .data\n");
exit(1);
}
#ifdef binary
fread(&leng, sizeof(int), 1, fp);
#else
fscanf(fp,"%d",&leng);
#endif
fclose(fp);
}
leng = AZ_gsum_int(leng, proc_config);
N_grid_pts=leng/num_PDE_eqns;
/* initialize the list of global indices. NOTE: the list of global */
/* indices must be in ascending order so that subsequent calls to */
/* AZ_find_index() will function properly. */
AZ_read_update(&N_update, &update, proc_config, N_grid_pts, num_PDE_eqns,
AZ_linear);
AZ_read_msr_matrix(update, &val, &bindx, N_update, proc_config);
AZ_transform(proc_config, &external, bindx, val, update, &update_index,
&extern_index, &data_org, N_update, 0, 0, 0, &cpntr,
AZ_MSR_MATRIX);
Amat = AZ_matrix_create( leng );
AZ_set_MSR(Amat, bindx, val, data_org, 0, NULL, AZ_LOCAL);
Amat->matrix_type = data_org[AZ_matrix_type];
data_org[AZ_N_rows] = data_org[AZ_N_internal] + data_org[AZ_N_border];
start_time = AZ_second();
AZ_defaults(options, params);
/*
scaling = AZ_scaling_create();
xxx = (double *) calloc( leng,sizeof(double));
rhs=(double *)calloc(leng,sizeof(double));
options[AZ_scaling] = AZ_sym_diag;
options[AZ_precond] = AZ_none;
options[AZ_max_iter] = 30;
options[AZ_keep_info] = 1;
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
don't forget vector rescaling ...
free(xxx);
free(rhs);
*/
options[AZ_scaling] = AZ_none;
ML_Create(&ml, N_levels);
/* set up discretization matrix and matrix vector function */
AZ_ML_Set_Amat(ml, N_levels-1, N_update, N_update, Amat, proc_config);
ML_Aggregate_Create( &ag );
Nrigid = 0;
if (proc_config[AZ_node] == 0) {
sprintf(filename,"rigid_body_mode%d",Nrigid+1);
while( (fp = fopen(filename,"r")) != NULL) {
fclose(fp);
Nrigid++;
sprintf(filename,"rigid_body_mode%d",Nrigid+1);
}
}
Nrigid = AZ_gsum_int(Nrigid,proc_config);
if (Nrigid != 0) {
rigid = (double *) ML_allocate( sizeof(double)*Nrigid*(N_update+1) );
if (rigid == NULL) {
printf("Error: Not enough space for rigid body modes\n");
}
}
rhs=(double *)malloc(leng*sizeof(double));
AZ_random_vector(rhs, data_org, proc_config);
for (i = 0; i < Nrigid; i++) {
sprintf(filename,"rigid_body_mode%d",i+1);
AZ_input_msr_matrix(filename, update, &mode, &garbage,
N_update, proc_config);
/*
AZ_sym_rescale_sl(mode, Amat->data_org, options, proc_config, scaling);
*/
/*
Amat->matvec(mode, rigid, Amat, proc_config);
for (j = 0; j < N_update; j++) printf("this is %d %e\n",j,rigid[j]);
*/
for (j = 0; j < i; j++) {
alpha = -AZ_gdot(N_update, mode, &(rigid[j*N_update]), proc_config)/AZ_gdot(N_update, &(rigid[j*N_update]), &(rigid[j*N_update]), proc_config);
daxpy_(&N_update, &alpha, &(rigid[j*N_update]), &one, mode, &one);
printf("alpha1 is %e\n",alpha);
}
alpha = -AZ_gdot(N_update, mode, rhs, proc_config)/AZ_gdot(N_update, mode, mode, proc_config);
printf("alpha2 is %e\n",alpha);
daxpy_(&N_update, &alpha, mode, &one, rhs, &one);
for (j = 0; j < N_update; j++) rigid[i*N_update+j] = mode[j];
free(mode);
free(garbage);
}
for (j = 0; j < Nrigid; j++) {
alpha = -AZ_gdot(N_update, rhs, &(rigid[j*N_update]), proc_config)/AZ_gdot(N_update, &(rigid[j*N_update]), &(rigid[j*N_update]), proc_config);
daxpy_(&N_update, &alpha, &(rigid[j*N_update]), &one, rhs, &one);
printf("alpha4 is %e\n",alpha);
}
for (i = 0; i < Nrigid; i++) {
alpha = -AZ_gdot(N_update, &(rigid[i*N_update]), rhs, proc_config);
printf("alpha is %e\n",alpha);
}
if (Nrigid != 0) {
ML_Aggregate_Set_NullSpace(ag, num_PDE_eqns, Nrigid, rigid, N_update);
/*
free(rigid);
*/
}
coarsest_level = ML_Gen_MGHierarchy_UsingAggregation(ml, N_levels-1, ML_DECREASING, ag);
coarsest_level = N_levels - coarsest_level;
/*
ML_Operator_Print(&(ml->Pmat[N_levels-2]), "Pmat");
exit(1);
*/
if ( proc_config[AZ_node] == 0 )
printf("Coarse level = %d \n", coarsest_level);
/* set up smoothers */
for (level = N_levels-1; level > coarsest_level; level--) {
j = 10;
if (level == N_levels-1) j = 10;
options[AZ_solver] = AZ_cg;
options[AZ_precond]=AZ_sym_GS; options[AZ_subdomain_solve]=AZ_icc;
/*
options[AZ_precond] = AZ_none;
*/
options[AZ_poly_ord] = 5;
ML_Gen_SmootherAztec(ml, level, options, params, proc_config, status,
j, ML_PRESMOOTHER,NULL);
ML_Gen_SmootherAztec(ml, level, options, params, proc_config, status,
j, ML_POSTSMOOTHER,NULL);
/*
ML_Gen_Smoother_SymGaussSeidel(ml , level, ML_PRESMOOTHER, nsmooth,1.0);
ML_Gen_Smoother_SymGaussSeidel(ml , level, ML_POSTSMOOTHER, nsmooth,1.0);
*/
/*
nblocks = ML_Aggregate_Get_AggrCount( ag, level );
ML_Aggregate_Get_AggrMap( ag, level, &blocks);
ML_Gen_Smoother_VBlockSymGaussSeidel( ml , level, ML_BOTH, nsmooth, 1.0,
nblocks, blocks);
ML_Gen_Smoother_VBlockSymGaussSeidel( ml , level, ML_POSTSMOOTHER, nsmooth, 1.0,
nblocks, blocks);
*/
/*
ML_Gen_Smoother_VBlockJacobi( ml , level, ML_PRESMOOTHER, nsmooth, .5,
nblocks, blocks);
ML_Gen_Smoother_VBlockJacobi( ml , level, ML_POSTSMOOTHER, nsmooth,.5,
nblocks, blocks);
*/
/*
ML_Gen_Smoother_GaussSeidel(ml , level, ML_PRESMOOTHER, nsmooth);
ML_Gen_Smoother_GaussSeidel(ml , level, ML_POSTSMOOTHER, nsmooth);
*/
/*
need to change this when num_pdes is different on different levels
*/
/*
if (level == N_levels-1) {
ML_Gen_Smoother_BlockGaussSeidel(ml , level, ML_PRESMOOTHER, nsmooth, 0.5, num_PDE_eqns);
ML_Gen_Smoother_BlockGaussSeidel(ml , level, ML_POSTSMOOTHER, nsmooth, 0.5, num_PDE_eqns);
}
else {
ML_Gen_Smoother_BlockGaussSeidel(ml , level, ML_PRESMOOTHER, nsmooth, 0.5, 2*num_PDE_eqns);
ML_Gen_Smoother_BlockGaussSeidel(ml , level, ML_POSTSMOOTHER, nsmooth, 0.5, 2*num_PDE_eqns);
}
*/
/*
*/
/*
ML_Gen_SmootherJacobi(ml , level, ML_PRESMOOTHER, nsmooth, .67);
ML_Gen_SmootherJacobi(ml , level, ML_POSTSMOOTHER, nsmooth, .67 );
*/
}
/*
ML_Gen_CoarseSolverSuperLU( ml, coarsest_level);
*/
/*
ML_Gen_SmootherSymGaussSeidel(ml , coarsest_level, ML_PRESMOOTHER, 2*nsmooth,1.);
*/
/*
ML_Gen_SmootherBlockGaussSeidel(ml , level, ML_PRESMOOTHER, 50*nsmooth, 1.0, 2*num_PDE_eqns);
*/
ML_Gen_Smoother_BlockGaussSeidel(ml , level, ML_PRESMOOTHER, 2*nsmooth, 1.0, num_PDE_eqns);
ML_Gen_Solver(ml, ML_MGV, N_levels-1, coarsest_level);
AZ_defaults(options, params);
options[AZ_solver] = AZ_GMRESR;
options[AZ_scaling] = AZ_none;
options[AZ_precond] = AZ_user_precond;
options[AZ_conv] = AZ_rhs;
options[AZ_output] = 1;
options[AZ_max_iter] = 1500;
options[AZ_poly_ord] = 5;
options[AZ_kspace] = 130;
params[AZ_tol] = 1.0e-8;
AZ_set_ML_preconditioner(&Pmat, Amat, ml, options);
setup_time = AZ_second() - start_time;
xxx = (double *) malloc( leng*sizeof(double));
/* Set rhs */
fp = fopen("AZ_capture_rhs.dat","r");
if (fp == NULL) {
if (proc_config[AZ_node] == 0) printf("taking random vector for rhs\n");
/*
AZ_random_vector(rhs, data_org, proc_config);
AZ_reorder_vec(rhs, data_org, update_index, NULL);
AZ_random_vector(xxx, data_org, proc_config);
AZ_reorder_vec(xxx, data_org, update_index, NULL);
Amat->matvec(xxx, rhs, Amat, proc_config);
*/
}
else {
ch = getc(fp);
if (ch == 'S') {
while ( (ch = getc(fp)) != '\n') ;
}
else ungetc(ch,fp);
for (i = 0; i < data_org[AZ_N_internal]+data_org[AZ_N_border]; i++)
fscanf(fp,"%lf",&(rhs[i]));
fclose(fp);
}
for (iii = 0; iii < leng; iii++) xxx[iii] = 0.0;
/* Set x */
fp = fopen("AZ_capture_init_guess.dat","r");
if (fp != NULL) {
ch = getc(fp);
if (ch == 'S') {
while ( (ch = getc(fp)) != '\n') ;
}
else ungetc(ch,fp);
for (i = 0; i < data_org[AZ_N_internal]+data_org[AZ_N_border]; i++)
fscanf(fp,"%lf",&(xxx[i]));
fclose(fp);
options[AZ_conv] = AZ_expected_values;
}
/* if Dirichlet BC ... put the answer in */
for (i = 0; i < data_org[AZ_N_internal]+data_org[AZ_N_border]; i++) {
if ( (val[i] > .99999999) && (val[i] < 1.0000001))
xxx[i] = rhs[i];
}
fp = fopen("AZ_no_multilevel.dat","r");
scaling = AZ_scaling_create();
start_time = AZ_second();
if (fp != NULL) {
fclose(fp);
options[AZ_precond] = AZ_none;
options[AZ_scaling] = AZ_sym_diag;
options[AZ_ignore_scaling] = AZ_TRUE;
options[AZ_keep_info] = 1;
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
/*
options[AZ_pre_calc] = AZ_reuse;
options[AZ_conv] = AZ_expected_values;
if (proc_config[AZ_node] == 0)
printf("\n-------- Second solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
if (proc_config[AZ_node] == 0)
printf("\n-------- Third solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
*/
}
else {
options[AZ_keep_info] = 1;
/*
options[AZ_max_iter] = 40;
*/
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
for (j = 0; j < Nrigid; j++) {
alpha = -AZ_gdot(N_update, xxx, &(rigid[j*N_update]), proc_config)/AZ_gdot(N_update, &(rigid[j*N_update]), &(rigid[j*N_update]), proc_config);
daxpy_(&N_update, &alpha, &(rigid[j*N_update]), &one, xxx, &one);
printf("alpha5 is %e\n",alpha);
}
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
options[AZ_pre_calc] = AZ_reuse;
options[AZ_conv] = AZ_expected_values;
/*
if (proc_config[AZ_node] == 0)
printf("\n-------- Second solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
if (proc_config[AZ_node] == 0)
printf("\n-------- Third solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
*/
}
solve_time = AZ_second() - start_time;
if (proc_config[AZ_node] == 0)
printf("Solve time = %e, MG Setup time = %e\n", solve_time, setup_time);
ML_Aggregate_Destroy(&ag);
ML_Destroy(&ml);
AZ_free((void *) Amat->data_org);
AZ_free((void *) Amat->val);
AZ_free((void *) Amat->bindx);
AZ_free((void *) update);
AZ_free((void *) external);
AZ_free((void *) extern_index);
AZ_free((void *) update_index);
if (Amat != NULL) AZ_matrix_destroy(&Amat);
if (Pmat != NULL) AZ_precond_destroy(&Pmat);
free(xxx);
free(rhs);
#ifdef ML_MPI
MPI_Finalize();
#endif
return 0;
}
int main(int argc, char *argv[])
{
int num_PDE_eqns=6, N_levels=4, nsmooth=2;
int leng, level, N_grid_pts, coarsest_level;
/* See Aztec User's Guide for more information on the */
/* variables that follow. */
int proc_config[AZ_PROC_SIZE], options[AZ_OPTIONS_SIZE];
double params[AZ_PARAMS_SIZE], status[AZ_STATUS_SIZE];
/* data structure for matrix corresponding to the fine grid */
double *val = NULL, *xxx, *rhs, solve_time, setup_time, start_time;
AZ_MATRIX *Amat;
AZ_PRECOND *Pmat = NULL;
ML *ml;
FILE *fp;
int i, j, Nrigid, *garbage = NULL;
#ifdef ML_partition
int nblocks;
int *block_list = NULL;
int k;
#endif
struct AZ_SCALING *scaling;
ML_Aggregate *ag;
double *mode, *rigid;
char filename[80];
double alpha;
int allocated = 0;
int old_prec, old_sol;
double old_tol;
/*
double *Amode, beta, biggest;
int big_ind = -1, ii;
*/
ML_Operator *Amatrix;
int *rowi_col = NULL, rowi_N, count2, ccc;
double *rowi_val = NULL;
double max_diag, min_diag, max_sum, sum;
int nBlocks, *blockIndices, Ndof;
#ifdef ML_partition
FILE *fp2;
int count;
if (argc != 2) {
printf("Usage: ml_read_elas num_processors\n");
exit(1);
}
else sscanf(argv[1],"%d",&nblocks);
#endif
#ifdef HAVE_MPI
MPI_Init(&argc,&argv);
/* get number of processors and the name of this processor */
AZ_set_proc_config(proc_config, MPI_COMM_WORLD);
#else
AZ_set_proc_config(proc_config, AZ_NOT_MPI);
#endif
/* read in the number of matrix equations */
leng = 0;
if (proc_config[AZ_node] == 0) {
# ifdef binary
fp=fopen(".data","rb");
# else
fp=fopen(".data","r");
# endif
if (fp==NULL) {
printf("couldn't open file .data\n");
exit(1);
}
# ifdef binary
fread(&leng, sizeof(int), 1, fp);
# else
fscanf(fp,"%d",&leng);
# endif
fclose(fp);
}
leng = AZ_gsum_int(leng, proc_config);
N_grid_pts=leng/num_PDE_eqns;
/* initialize the list of global indices. NOTE: the list of global */
/* indices must be in ascending order so that subsequent calls to */
/* AZ_find_index() will function properly. */
if (proc_config[AZ_N_procs] == 1) i = AZ_linear;
else i = AZ_file;
AZ_read_update(&N_update, &update, proc_config, N_grid_pts, num_PDE_eqns,i);
AZ_read_msr_matrix(update, &val, &bindx, N_update, proc_config);
/* This code is to fix things up so that we are sure we have */
/* all block (including the ghost nodes the same size. */
AZ_block_MSR(&bindx, &val, N_update, num_PDE_eqns, update);
AZ_transform_norowreordering(proc_config, &external, bindx, val, update, &update_index,
&extern_index, &data_org, N_update, 0, 0, 0, &cpntr,
AZ_MSR_MATRIX);
Amat = AZ_matrix_create( leng );
AZ_set_MSR(Amat, bindx, val, data_org, 0, NULL, AZ_LOCAL);
Amat->matrix_type = data_org[AZ_matrix_type];
data_org[AZ_N_rows] = data_org[AZ_N_internal] + data_org[AZ_N_border];
#ifdef SCALE_ME
ML_MSR_sym_diagonal_scaling(Amat, proc_config, &scaling_vect);
#endif
start_time = AZ_second();
options[AZ_scaling] = AZ_none;
ML_Create(&ml, N_levels);
ML_Set_PrintLevel(10);
/* set up discretization matrix and matrix vector function */
AZ_ML_Set_Amat(ml, N_levels-1, N_update, N_update, Amat, proc_config);
#ifdef ML_partition
/* this code is meant to partition the matrices so that things can be */
/* run in parallel later. */
/* It is meant to be run on only one processor. */
#ifdef MB_MODIF
fp2 = fopen(".update","w");
#else
fp2 = fopen("partition_file","w");
#endif
ML_Operator_AmalgamateAndDropWeak(&(ml->Amat[N_levels-1]), num_PDE_eqns, 0.0);
ML_Gen_Blocks_Metis(ml, N_levels-1, &nblocks, &block_list);
for (i = 0; i < nblocks; i++) {
count = 0;
for (j = 0; j < ml->Amat[N_levels-1].outvec_leng; j++) {
if (block_list[j] == i) count++;
}
fprintf(fp2," %d\n",count*num_PDE_eqns);
for (j = 0; j < ml->Amat[N_levels-1].outvec_leng; j++) {
if (block_list[j] == i) {
for (k = 0; k < num_PDE_eqns; k++) fprintf(fp2,"%d\n",j*num_PDE_eqns+k);
}
}
}
fclose(fp2);
ML_Operator_UnAmalgamateAndDropWeak(&(ml->Amat[N_levels-1]),num_PDE_eqns,0.0);
#ifdef MB_MODIF
printf(" partition file dumped in .update\n");
#endif
exit(1);
#endif
ML_Aggregate_Create( &ag );
/*
ML_Aggregate_Set_CoarsenScheme_MIS(ag);
*/
#ifdef MB_MODIF
ML_Aggregate_Set_DampingFactor(ag,1.50);
#else
ML_Aggregate_Set_DampingFactor(ag,1.5);
#endif
ML_Aggregate_Set_CoarsenScheme_METIS(ag);
ML_Aggregate_Set_NodesPerAggr( ml, ag, -1, 35);
/*
ML_Aggregate_Set_Phase3AggregateCreationAggressiveness(ag, 10.001);
*/
ML_Aggregate_Set_Threshold(ag, 0.0);
ML_Aggregate_Set_MaxCoarseSize( ag, 300);
/* read in the rigid body modes */
Nrigid = 0;
/* to ensure compatibility with RBM dumping software */
if (proc_config[AZ_node] == 0) {
sprintf(filename,"rigid_body_mode%02d",Nrigid+1);
while( (fp = fopen(filename,"r")) != NULL) {
which_filename = 1;
fclose(fp);
Nrigid++;
sprintf(filename,"rigid_body_mode%02d",Nrigid+1);
}
sprintf(filename,"rigid_body_mode%d",Nrigid+1);
while( (fp = fopen(filename,"r")) != NULL) {
fclose(fp);
Nrigid++;
sprintf(filename,"rigid_body_mode%d",Nrigid+1);
}
}
Nrigid = AZ_gsum_int(Nrigid,proc_config);
if (Nrigid != 0) {
rigid = (double *) ML_allocate( sizeof(double)*Nrigid*(N_update+1) );
if (rigid == NULL) {
printf("Error: Not enough space for rigid body modes\n");
}
}
rhs = (double *) malloc(leng*sizeof(double));
xxx = (double *) malloc(leng*sizeof(double));
for (iii = 0; iii < leng; iii++) xxx[iii] = 0.0;
for (i = 0; i < Nrigid; i++) {
if (which_filename == 1) sprintf(filename,"rigid_body_mode%02d",i+1);
else sprintf(filename,"rigid_body_mode%d",i+1);
AZ_input_msr_matrix(filename,update,&mode,&garbage,N_update,proc_config);
AZ_reorder_vec(mode, data_org, update_index, NULL);
/* here is something to stick a rigid body mode as the initial */
/* The idea is to solve A x = 0 without smoothing with a two */
/* level method. If everything is done properly, we should */
/* converge in 2 iterations. */
/* Note: we must also zero out components of the rigid body */
/* mode that correspond to Dirichlet bcs. */
if (i == -4) {
for (iii = 0; iii < leng; iii++) xxx[iii] = mode[iii];
ccc = 0;
Amatrix = &(ml->Amat[N_levels-1]);
for (iii = 0; iii < Amatrix->outvec_leng; iii++) {
ML_get_matrix_row(Amatrix,1,&iii,&allocated,&rowi_col,&rowi_val,
&rowi_N, 0);
count2 = 0;
for (j = 0; j < rowi_N; j++) if (rowi_val[j] != 0.) count2++;
if (count2 <= 1) { xxx[iii] = 0.; ccc++; }
}
free(rowi_col); free(rowi_val);
allocated = 0; rowi_col = NULL; rowi_val = NULL;
}
/*
* Rescale matrix/rigid body modes and checking
*
AZ_sym_rescale_sl(mode, Amat->data_org, options, proc_config, scaling);
Amat->matvec(mode, rigid, Amat, proc_config);
for (j = 0; j < N_update; j++) printf("this is %d %e\n",j,rigid[j]);
*/
/* Here is some code to check that the rigid body modes are */
/* really rigid body modes. The idea is to multiply by A and */
/* then to zero out things that we "think" are boundaries. */
/* In this hardwired example, things near boundaries */
/* correspond to matrix rows that do not have 81 nonzeros. */
/*
Amode = (double *) malloc(leng*sizeof(double));
Amat->matvec(mode, Amode, Amat, proc_config);
j = 0;
biggest = 0.0;
for (ii = 0; ii < N_update; ii++) {
if ( Amat->bindx[ii+1] - Amat->bindx[ii] != 80) {
Amode[ii] = 0.; j++;
}
else {
if ( fabs(Amode[ii]) > biggest) {
biggest=fabs(Amode[ii]); big_ind = ii;
}
}
}
printf("%d entries zeroed out of %d elements\n",j,N_update);
alpha = AZ_gdot(N_update, Amode, Amode, proc_config);
beta = AZ_gdot(N_update, mode, mode, proc_config);
printf("||A r||^2 =%e, ||r||^2 = %e, ratio = %e\n",
alpha,beta,alpha/beta);
printf("the biggest is %e at row %d\n",biggest,big_ind);
free(Amode);
*/
/* orthogonalize mode with respect to previous modes. */
for (j = 0; j < i; j++) {
alpha = -AZ_gdot(N_update, mode, &(rigid[j*N_update]), proc_config)/
AZ_gdot(N_update, &(rigid[j*N_update]),
&(rigid[j*N_update]), proc_config);
/* daxpy_(&N_update,&alpha,&(rigid[j*N_update]), &one, mode, &one); */
}
#ifndef MB_MODIF
printf(" after mb %e %e %e\n",mode[0],mode[1],mode[2]);
#endif
for (j = 0; j < N_update; j++) rigid[i*N_update+j] = mode[j];
free(mode);
free(garbage); garbage = NULL;
}
if (Nrigid != 0) {
ML_Aggregate_Set_BlockDiagScaling(ag);
ML_Aggregate_Set_NullSpace(ag, num_PDE_eqns, Nrigid, rigid, N_update);
free(rigid);
}
#ifdef SCALE_ME
ML_Aggregate_Scale_NullSpace(ag, scaling_vect, N_update);
#endif
coarsest_level = ML_Gen_MGHierarchy_UsingAggregation(ml, N_levels-1,
ML_DECREASING, ag);
AZ_defaults(options, params);
coarsest_level = N_levels - coarsest_level;
if ( proc_config[AZ_node] == 0 )
printf("Coarse level = %d \n", coarsest_level);
/* set up smoothers */
for (level = N_levels-1; level > coarsest_level; level--) {
/*
ML_Gen_Smoother_BlockGaussSeidel(ml, level,ML_BOTH, 1, 1., num_PDE_eqns);
*/
/* Sparse approximate inverse smoother that acutally does both */
/* pre and post smoothing. */
/*
ML_Gen_Smoother_ParaSails(ml , level, ML_PRESMOOTHER, nsmooth,
parasails_sym, parasails_thresh,
parasails_nlevels, parasails_filter,
parasails_loadbal, parasails_factorized);
*/
/* This is the symmetric Gauss-Seidel smoothing that we usually use. */
/* In parallel, it is not a true Gauss-Seidel in that each processor */
/* does a Gauss-Seidel on its local submatrix independent of the */
/* other processors. */
/* ML_Gen_Smoother_Cheby(ml, level, ML_BOTH, 30., nsmooth); */
Ndof = ml->Amat[level].invec_leng;
ML_Gen_Blocks_Aggregates(ag, level, &nBlocks, &blockIndices);
ML_Gen_Smoother_BlockDiagScaledCheby(ml, level, ML_BOTH, 30.,nsmooth,
nBlocks, blockIndices);
/*
ML_Gen_Smoother_SymGaussSeidel(ml , level, ML_BOTH, nsmooth,1.);
*/
/* This is a true Gauss Seidel in parallel. This seems to work for */
/* elasticity problems. However, I don't believe that this is very */
/* efficient in parallel. */
/*
nblocks = ml->Amat[level].invec_leng/num_PDE_eqns;
blocks = (int *) ML_allocate(sizeof(int)*N_update);
for (i =0; i < ml->Amat[level].invec_leng; i++)
blocks[i] = i/num_PDE_eqns;
ML_Gen_Smoother_VBlockSymGaussSeidelSequential(ml , level, ML_PRESMOOTHER,
nsmooth, 1., nblocks, blocks);
ML_Gen_Smoother_VBlockSymGaussSeidelSequential(ml, level, ML_POSTSMOOTHER,
nsmooth, 1., nblocks, blocks);
free(blocks);
*/
/* Block Jacobi Smoothing */
/*
nblocks = ml->Amat[level].invec_leng/num_PDE_eqns;
blocks = (int *) ML_allocate(sizeof(int)*N_update);
for (i =0; i < ml->Amat[level].invec_leng; i++)
blocks[i] = i/num_PDE_eqns;
ML_Gen_Smoother_VBlockJacobi(ml , level, ML_BOTH, nsmooth,
ML_ONE_STEP_CG, nblocks, blocks);
free(blocks);
*/
/* Jacobi Smoothing */
/*
ML_Gen_Smoother_Jacobi(ml , level, ML_PRESMOOTHER, nsmooth, ML_ONE_STEP_CG);
ML_Gen_Smoother_Jacobi(ml , level, ML_POSTSMOOTHER, nsmooth,ML_ONE_STEP_CG);
*/
/* This does a block Gauss-Seidel (not true GS in parallel) */
/* where each processor has 'nblocks' blocks. */
/*
nblocks = 250;
ML_Gen_Blocks_Metis(ml, level, &nblocks, &blocks);
ML_Gen_Smoother_VBlockJacobi(ml , level, ML_BOTH, nsmooth,ML_ONE_STEP_CG,
nblocks, blocks);
free(blocks);
*/
num_PDE_eqns = 6;
}
/* Choose coarse grid solver: mls, superlu, symGS, or Aztec */
/*
ML_Gen_Smoother_Cheby(ml, coarsest_level, ML_BOTH, 30., nsmooth);
ML_Gen_CoarseSolverSuperLU( ml, coarsest_level);
*/
/*
ML_Gen_Smoother_SymGaussSeidel(ml , coarsest_level, ML_BOTH, nsmooth,1.);
*/
old_prec = options[AZ_precond];
old_sol = options[AZ_solver];
old_tol = params[AZ_tol];
params[AZ_tol] = 1.0e-9;
params[AZ_tol] = 1.0e-5;
options[AZ_precond] = AZ_Jacobi;
options[AZ_solver] = AZ_cg;
options[AZ_poly_ord] = 1;
options[AZ_conv] = AZ_r0;
options[AZ_orth_kvecs] = AZ_TRUE;
j = AZ_gsum_int(ml->Amat[coarsest_level].outvec_leng, proc_config);
options[AZ_keep_kvecs] = j - 6;
options[AZ_max_iter] = options[AZ_keep_kvecs];
ML_Gen_SmootherAztec(ml, coarsest_level, options, params,
proc_config, status, options[AZ_keep_kvecs], ML_PRESMOOTHER, NULL);
options[AZ_conv] = AZ_noscaled;
options[AZ_keep_kvecs] = 0;
options[AZ_orth_kvecs] = 0;
options[AZ_precond] = old_prec;
options[AZ_solver] = old_sol;
params[AZ_tol] = old_tol;
/* */
#ifdef RST_MODIF
ML_Gen_Solver(ml, ML_MGV, N_levels-1, coarsest_level);
#else
#ifdef MB_MODIF
ML_Gen_Solver(ml, ML_SAAMG, N_levels-1, coarsest_level);
#else
ML_Gen_Solver(ml, ML_MGFULLV, N_levels-1, coarsest_level);
#endif
#endif
options[AZ_solver] = AZ_GMRESR;
options[AZ_solver] = AZ_cg;
options[AZ_scaling] = AZ_none;
options[AZ_precond] = AZ_user_precond;
options[AZ_conv] = AZ_r0;
options[AZ_conv] = AZ_noscaled;
options[AZ_output] = 1;
options[AZ_max_iter] = 500;
options[AZ_poly_ord] = 5;
options[AZ_kspace] = 40;
params[AZ_tol] = 4.8e-6;
AZ_set_ML_preconditioner(&Pmat, Amat, ml, options);
setup_time = AZ_second() - start_time;
/* Set rhs */
fp = fopen("AZ_capture_rhs.dat","r");
if (fp == NULL) {
AZ_random_vector(rhs, data_org, proc_config);
if (proc_config[AZ_node] == 0) printf("taking random vector for rhs\n");
for (i = 0; i < -N_update; i++) {
rhs[i] = (double) update[i]; rhs[i] = 7.;
}
}
else {
if (proc_config[AZ_node]== 0) printf("reading rhs guess from file\n");
AZ_input_msr_matrix("AZ_capture_rhs.dat", update, &rhs, &garbage,
N_update, proc_config);
free(garbage);
}
AZ_reorder_vec(rhs, data_org, update_index, NULL);
printf("changing rhs by multiplying with A\n");
Amat->matvec(rhs, xxx, Amat, proc_config);
for (i = 0; i < N_update; i++) rhs[i] = xxx[i];
fp = fopen("AZ_capture_init_guess.dat","r");
if (fp != NULL) {
fclose(fp);
if (proc_config[AZ_node]== 0) printf("reading initial guess from file\n");
AZ_input_msr_matrix("AZ_capture_init_guess.dat", update, &xxx, &garbage,
N_update, proc_config);
free(garbage);
xxx = (double *) realloc(xxx, sizeof(double)*(
Amat->data_org[AZ_N_internal]+
Amat->data_org[AZ_N_border] +
Amat->data_org[AZ_N_external]));
}
AZ_reorder_vec(xxx, data_org, update_index, NULL);
/* if Dirichlet BC ... put the answer in */
/*
for (i = 0; i < data_org[AZ_N_internal]+data_org[AZ_N_border]; i++) {
if ( (val[i] > .99999999) && (val[i] < 1.0000001))
xxx[i] = rhs[i];
}
*/
fp = fopen("AZ_no_multilevel.dat","r");
scaling = AZ_scaling_create();
start_time = AZ_second();
if (fp != NULL) {
fclose(fp);
options[AZ_precond] = AZ_none;
options[AZ_scaling] = AZ_sym_diag;
options[AZ_ignore_scaling] = AZ_TRUE;
options[AZ_keep_info] = 1;
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
/*
options[AZ_pre_calc] = AZ_reuse;
options[AZ_conv] = AZ_expected_values;
if (proc_config[AZ_node] == 0)
printf("\n-------- Second solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
if (proc_config[AZ_node] == 0)
printf("\n-------- Third solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling);
*/
}
else {
options[AZ_keep_info] = 1;
options[AZ_conv] = AZ_noscaled;
options[AZ_conv] = AZ_r0;
params[AZ_tol] = 1.0e-7;
/* ML_Iterate(ml, xxx, rhs); */
alpha = sqrt(AZ_gdot(N_update, xxx, xxx, proc_config));
printf("init guess = %e\n",alpha);
alpha = sqrt(AZ_gdot(N_update, rhs, rhs, proc_config));
printf("rhs = %e\n",alpha);
#ifdef SCALE_ME
ML_MSR_scalerhs(rhs, scaling_vect, data_org[AZ_N_internal] +
data_org[AZ_N_border]);
ML_MSR_scalesol(xxx, scaling_vect, data_org[AZ_N_internal] +
data_org[AZ_N_border]);
#endif
max_diag = 0.;
min_diag = 1.e30;
max_sum = 0.;
for (i = 0; i < N_update; i++) {
if (Amat->val[i] < 0.) printf("woops negative diagonal A(%d,%d) = %e\n",
i,i,Amat->val[i]);
if (Amat->val[i] > max_diag) max_diag = Amat->val[i];
if (Amat->val[i] < min_diag) min_diag = Amat->val[i];
sum = fabs(Amat->val[i]);
for (j = Amat->bindx[i]; j < Amat->bindx[i+1]; j++) {
sum += fabs(Amat->val[j]);
}
if (sum > max_sum) max_sum = sum;
}
printf("Largest diagonal = %e, min diag = %e large abs row sum = %e\n",
max_diag, min_diag, max_sum);
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
options[AZ_pre_calc] = AZ_reuse;
options[AZ_conv] = AZ_expected_values;
/*
if (proc_config[AZ_node] == 0)
printf("\n-------- Second solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
if (proc_config[AZ_node] == 0)
printf("\n-------- Third solve with improved convergence test -----\n");
AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling);
*/
}
solve_time = AZ_second() - start_time;
if (proc_config[AZ_node] == 0)
printf("Solve time = %e, MG Setup time = %e\n", solve_time, setup_time);
if (proc_config[AZ_node] == 0)
printf("Printing out a few entries of the solution ...\n");
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 7) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
j = AZ_gsum_int(7, proc_config); /* sync processors */
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 23) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
j = AZ_gsum_int(7, proc_config); /* sync processors */
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 47) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
j = AZ_gsum_int(7, proc_config); /* sync processors */
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 101) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
j = AZ_gsum_int(7, proc_config); /* sync processors */
for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
if (update[j] == 171) {printf("solution(gid = %d) = %10.4e\n",
update[j],xxx[update_index[j]]); fflush(stdout);}
ML_Aggregate_Destroy(&ag);
ML_Destroy(&ml);
AZ_free((void *) Amat->data_org);
AZ_free((void *) Amat->val);
AZ_free((void *) Amat->bindx);
AZ_free((void *) update);
AZ_free((void *) external);
AZ_free((void *) extern_index);
AZ_free((void *) update_index);
AZ_scaling_destroy(&scaling);
if (Amat != NULL) AZ_matrix_destroy(&Amat);
if (Pmat != NULL) AZ_precond_destroy(&Pmat);
free(xxx);
free(rhs);
#ifdef HAVE_MPI
MPI_Finalize();
#endif
return 0;
}
/*----------------------------------------------------------------------*
| Constructor (public) m.gee 01/05|
| IMPORTANT: |
| No matter on which level we are here, the vector xfine is ALWAYS |
| a fine grid vector here! |
| this is the constructor for the ismatrixfree==false case
*----------------------------------------------------------------------*/
ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel(
int level, int nlevel, int printlevel, ML* ml,
ML_Aggregate* ag,Epetra_CrsMatrix** P,
ML_NOX::Ml_Nox_Fineinterface& interface,
const Epetra_Comm& comm, const Epetra_Vector& xfine,
bool ismatrixfree, bool matfreelev0, bool isnlnCG,
int nitersCG, bool broyden, Epetra_CrsMatrix* Jac,
string fsmoothertype, string smoothertype,
string coarsesolvetype,
int nsmooth_fine, int nsmooth, int nsmooth_coarse,
double conv_normF, double conv_nupdate,
int conv_maxiter,int numPDE, int nullspdim)
: fineinterface_(interface),
comm_(comm)
{
level_ = level; // this level
nlevel_ = nlevel; // number of total levels
ml_printlevel_ = printlevel; // printlevel
ml_ = ml; // the global ML object
ag_ = ag; // the global ML_Aggregate object
thislevel_prec_ = 0; // this level's linear preconditioner
thislevel_ml_ = 0; // this level's local ML object
thislevel_ag_ = 0; // this level's local ML_Aggregate object
coarseinterface_ = 0; // this level's coarse interface
coarseprepost_ = 0;
xthis_ = 0; // this level's current solution matching this level's map!!!!
thislevel_A_ = 0; // this level's NOX Matrixfree operator
SmootherA_ = 0; // this level's Epetra_CrsMatrix for thislevel_prec_
ismatrixfree_ = ismatrixfree; // matrixfree flag
conv_normF_ = conv_normF; // NOX convergence test stuff
conv_nupdate_ = conv_nupdate;
conv_maxiter_ = conv_maxiter;
absresid_ = 0;
nupdate_ = 0;
fv_ = 0;
maxiters_ = 0;
combo1_ = 0;
combo2_ = 0;
thislevel_linSys_ = 0; // this level's NOX linear system
nlParams_ = 0; // NOX parameters
initialGuess_ = 0; // NOX initial guess
group_ = 0; // NOX group
solver_ = 0; // NOX solver
isnlnCG_ = isnlnCG;
azlinSys_ = 0;
clone_ = 0;
nitersCG_ = nitersCG;
broyden_ = broyden;
Broyd_ = 0;
if (ismatrixfree_==true)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ismatrixfree_==true on level " << level_ << "\n"
<< "**ERR**: in constructor for ismatrixfree_==false - case\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// ------------------------------------------------------------------------
// get the Jacobian of this level
const Epetra_CrsGraph* graph = 0;
// ------------------------------------------------------------------------
if (level_==0)
{
graph = fineinterface_.getGraph();
// On fine level this is the fineinterface's Jacobian
if (matfreelev0==false)
SmootherA_ = fineinterface_.getJacobian();
else if (matfreelev0==true && Jac)
SmootherA_ = Jac;
else
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: something weired happened\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
}
// ------------------------------------------------------------------------
else
{
// On coarse levels get Jacobian from hierarchy
// Note: On levels>0 SmootherA_ is a real copy of the Jacobian
int maxnnz=0;
double cputime=0.0;
ML_Operator2EpetraCrsMatrix(&(ml_->Amat[level_]), SmootherA_, maxnnz,
false, cputime);
SmootherA_->OptimizeStorage();
graph = &(SmootherA_->Graph());
}
// just to be save
if (!SmootherA_ || !graph)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: Smoother==NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// ------------------------------------------------------------------------
// generate this level's coarse interface
coarseinterface_ = new ML_NOX::Nox_CoarseProblem_Interface(
fineinterface_,level_,ml_printlevel_,
P,&(graph->RowMap()),nlevel_);
// ------------------------------------------------------------------------
// generate this level's coarse prepostoperator
if (level_==0)
coarseprepost_ = new ML_NOX::Ml_Nox_CoarsePrePostOperator(*coarseinterface_,
fineinterface_);
// ------------------------------------------------------------------------
// get the current solution to this level
xthis_ = coarseinterface_->restrict_fine_to_this(xfine);
// ------------------------------------------------------------------------
// create this level's preconditioner
// We use a 1-level ML-hierarchy for that
ML_Aggregate_Create(&thislevel_ag_);
ML_Create(&thislevel_ml_,1);
// set the Jacobian on level 0 of the local ml
EpetraMatrix2MLMatrix(thislevel_ml_,0,
(dynamic_cast<Epetra_RowMatrix*>(SmootherA_)));
// construct a 1-level ML-hierarchy on this level as a smoother
ML_Set_PrintLevel(ml_printlevel_);
ML_Aggregate_Set_CoarsenScheme_Uncoupled(thislevel_ag_);
ML_Aggregate_Set_DampingFactor(thislevel_ag_, 0.0);
ML_Aggregate_Set_Threshold(thislevel_ag_, 0.0);
ML_Aggregate_Set_MaxCoarseSize(thislevel_ag_,1);
ML_Aggregate_Set_NullSpace(thislevel_ag_,numPDE,nullspdim,NULL,
SmootherA_->NumMyRows());
int thislevel_nlevel = ML_Gen_MGHierarchy_UsingAggregation(thislevel_ml_,0,
ML_INCREASING,thislevel_ag_);
if (thislevel_nlevel != 1)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ML generated a local hierarchy of " << thislevel_nlevel << " on level " << level_ << "\n"
<< "**ERR**: this is supposed to be 1 Level only!\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// set the smoother
if (level_==0)
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,fsmoothertype,nsmooth_fine);
else if (level_ != nlevel_-1) // set the smoother from the input
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,smoothertype,nsmooth);
else // set the coarse solver from the input
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,coarsesolvetype,nsmooth_coarse);
// create this level's preconditioner class
ML_Epetra::MultiLevelOperator* ml_tmp = new ML_Epetra::MultiLevelOperator(
thislevel_ml_,comm_,
SmootherA_->OperatorDomainMap(),
SmootherA_->OperatorRangeMap());
thislevel_prec_ = new ML_NOX::ML_Nox_ConstrainedMultiLevelOperator(ml_tmp,*coarseinterface_);
if (!thislevel_prec_)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: thislevel_prec_==NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// ------------------------------------------------------------------------
// set up NOX on this level
// ------------------------------------------------------------------------
nlParams_ = new Teuchos::ParameterList();
Teuchos::ParameterList& printParams = nlParams_->sublist("Printing");
printParams.setParameter("MyPID", comm_.MyPID());
printParams.setParameter("Output Precision", 14);
printParams.setParameter("Output Processor", 0);
if (ml_printlevel_>9)
printParams.setParameter("Output Information",
NOX::Utils::OuterIteration +
NOX::Utils::Warning);
else if (ml_printlevel_>8)
printParams.setParameter("Output Information",
NOX::Utils::Warning);
else
printParams.setParameter("Output Information",0);
if (level_==0)
nlParams_->sublist("Solver Options").setParameter("User Defined Pre/Post Operator", *coarseprepost_);
nlParams_->setParameter("Nonlinear Solver", "Line Search Based");
Teuchos::ParameterList& searchParams = nlParams_->sublist("Line Search");
Teuchos::ParameterList* lsParamsptr = 0;
if (isnlnCG_)
{
searchParams.setParameter("Method", "NonlinearCG");
Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
dirParams.setParameter("Method", "NonlinearCG");
Teuchos::ParameterList& nlcgParams = dirParams.sublist("Nonlinear CG");
nlcgParams.setParameter("Restart Frequency", 10);
nlcgParams.setParameter("Precondition", "On");
nlcgParams.setParameter("Orthogonalize", "Polak-Ribiere");
//nlcgParams.setParameter("Orthogonalize", "Fletcher-Reeves");
Teuchos::ParameterList& lsParams = nlcgParams.sublist("Linear Solver");
lsParams.setParameter("Aztec Solver", "CG");
lsParams.setParameter("Max Iterations", 1);
lsParams.setParameter("Tolerance", 1e-11);
lsParams.setParameter("Output Frequency", 0);
lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
lsParams.setParameter("Preconditioner","User Defined");
}
else // Newton's method using ML-preconditioned Aztec as linear solver
{
searchParams.setParameter("Method", "Full Step");
// Sublist for direction
Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
dirParams.setParameter("Method", "Newton");
Teuchos::ParameterList& newtonParams = dirParams.sublist("Newton");
newtonParams.setParameter("Forcing Term Method", "Constant");
//newtonParams.setParameter("Forcing Term Method", "Type 1");
//newtonParams.setParameter("Forcing Term Method", "Type 2");
newtonParams.setParameter("Forcing Term Minimum Tolerance", 1.0e-6);
newtonParams.setParameter("Forcing Term Maximum Tolerance", 0.1);
Teuchos::ParameterList& lsParams = newtonParams.sublist("Linear Solver");
lsParamsptr = &lsParams;
lsParams.setParameter("Size of Krylov Subspace", 100);
lsParams.setParameter("Aztec Solver", "GMRES");
lsParams.setParameter("Max Iterations", nitersCG_);
lsParams.setParameter("Tolerance", conv_normF_); // FIXME? is this correct?
if (ml_printlevel_>8)
lsParams.setParameter("Output Frequency", 50);
else
lsParams.setParameter("Output Frequency", 0);
lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
lsParams.setParameter("Preconditioner","User Defined");
}
// create the initial guess
initialGuess_ = new NOX::Epetra::Vector(*xthis_, NOX::DeepCopy, true);
// NOTE: do not delete xthis_, it's used and destroyed by initialGuess_
// create the necessary interfaces
NOX::EpetraNew::Interface::Preconditioner* iPrec = 0;
NOX::EpetraNew::Interface::Required* iReq = 0;
NOX::EpetraNew::Interface::Jacobian* iJac = 0;
if (isnlnCG_)
{
// create the matrixfree operator used in the nlnCG
thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
// create the necessary interfaces
iPrec = 0;
iReq = coarseinterface_;
iJac = thislevel_A_;
// create the linear system
thislevel_linSys_ = new ML_NOX::Ml_Nox_LinearSystem(
*iJac,*thislevel_A_,*iPrec,
coarseinterface_,*thislevel_prec_,
*xthis_,ismatrixfree_,level_,ml_printlevel_);
// create the group
group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*thislevel_linSys_);
}
else // Modified Newton's method
{
if (!broyden_)
{
// create the necessary interfaces
iPrec = this;
iReq = coarseinterface_;
//iJac = this;
thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
// create the initial guess vector
//clone_ = new Epetra_Vector(*xthis_);
// create the linear system
//azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
// printParams,*lsParamsptr,
// *iJac,*SmootherA_,*iPrec,
// *thislevel_prec_,*clone_);
azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
printParams,*lsParamsptr,
*thislevel_A_,*thislevel_A_,*iPrec,
*thislevel_prec_,*xthis_);
}
else // use a Broyden update for the Jacobian
{
// create the initial guess vector
//clone_ = new Epetra_Vector(*xthis_);
// create the necessary interfaces
iPrec = this;
iReq = coarseinterface_;
Broyd_ = new NOX::EpetraNew::BroydenOperator(*nlParams_,*xthis_,
*SmootherA_,false);
// create the linear system
azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
printParams,*lsParamsptr,
*Broyd_,*SmootherA_,*iPrec,
*thislevel_prec_,*xthis_);
}
// create the group
group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,
*azlinSys_);
}
// create convergence test
create_Nox_Convergencetest(conv_normF_,conv_nupdate_,conv_maxiter_);
// create the solver
solver_ = new NOX::Solver::Manager(*group_,*combo2_,*nlParams_);
return;
}
/*----------------------------------------------------------------------*
| Constructor (public) m.gee 01/05|
| IMPORTANT: |
| No matter on which level we are here, the vector xfine is ALWAYS |
| a fine grid vector here! |
| this is the constructor for the ismatrixfree==true case
*----------------------------------------------------------------------*/
ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel(
int level, int nlevel, int printlevel, ML* ml,
ML_Aggregate* ag,Epetra_CrsMatrix** P,
ML_NOX::Ml_Nox_Fineinterface& interface,
const Epetra_Comm& comm, const Epetra_Vector& xfine,
bool ismatrixfree, bool isnlnCG, int nitersCG, bool broyden,
string fsmoothertype, string smoothertype, string coarsesolvetype,
int nsmooth_fine, int nsmooth, int nsmooth_coarse,
double conv_normF,
double conv_nupdate, int conv_maxiter,
int numPDE, int nullspdim, Epetra_CrsMatrix* Mat,
ML_NOX::Nox_CoarseProblem_Interface* coarseinterface)
: fineinterface_(interface),
comm_(comm)
{
level_ = level; // this level
nlevel_ = nlevel; // number of total levels
ml_printlevel_ = printlevel; // printlevel
ml_ = ml; // the global ML object
ag_ = ag; // the global ML_Aggregate object
thislevel_prec_ = 0; // this level's linear preconditioner
thislevel_ml_ = 0; // this level's local ML object
thislevel_ag_ = 0; // this level's local ML_Aggregate object
coarseinterface_ = coarseinterface; // this level's coarse interface
coarseprepost_ = 0;
xthis_ = 0; // this level's current solution matching this level's map!!!!
thislevel_A_ = 0; // this level's NOX Matrixfree operator
SmootherA_ = 0; // this level's Epetra_CrsMatrix for thislevel_prec_
ismatrixfree_ = ismatrixfree; // matrixfree flag
conv_normF_ = conv_normF; // NOX convergence test stuff
conv_nupdate_ = conv_nupdate;
conv_maxiter_ = conv_maxiter;
absresid_ = 0;
nupdate_ = 0;
fv_ = 0;
maxiters_ = 0;
combo1_ = 0;
combo2_ = 0;
thislevel_linSys_ = 0; // this level's NOX linear system
nlParams_ = 0; // NOX parameters
initialGuess_ = 0; // NOX initial guess
group_ = 0; // NOX group
solver_ = 0; // NOX solver
SmootherA_ = Mat;
isnlnCG_ = isnlnCG;
azlinSys_ = 0;
clone_ = 0;
nitersCG_ = nitersCG;
broyden_ = broyden;
Broyd_ = 0;
#if 0
if (isnlnCG_==false &&
(fsmoothertype == "Jacobi" ||
smoothertype == "Jacobi" ||
coarsesolvetype == "Jacobi" ))
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: Modified Newton's method not supported for \n"
<< "**ERR**: ismatrixfree_==true && smoothertype == Jacobi-Smoother\n"
<< "**ERR**: because no full Jacobian exists!\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
#endif
if (ismatrixfree_==false)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ismatrixfree_==false on level " << level_ << "\n"
<< "**ERR**: in constructor for ismatrixfree_==true - case\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
if (!coarseinterface_)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ptr to coarseinterface=NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
if (!Mat)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ptr to Matrix Mat=NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// ------------------------------------------------------------------------
Mat->OptimizeStorage();
// ------------------------------------------------------------------------
// get the current solution to this level
xthis_ = coarseinterface_->restrict_fine_to_this(xfine);
// ------------------------------------------------------------------------
// create this level's preconditioner
// We use a 1-level ML-hierarchy for that
ML_Aggregate_Create(&thislevel_ag_);
ML_Create(&thislevel_ml_,1);
// ------------------------------------------------------------------------
// set the Jacobian on level 0 of the local ml
EpetraMatrix2MLMatrix(thislevel_ml_,0,
(dynamic_cast<Epetra_RowMatrix*>(Mat)));
// ------------------------------------------------------------------------
// construct a 1-level ML-hierarchy on this level as a smoother
// ------------------------------------------------------------------------
ML_Set_PrintLevel(ml_printlevel_);
ML_Aggregate_Set_CoarsenScheme_Uncoupled(thislevel_ag_);
ML_Aggregate_Set_DampingFactor(thislevel_ag_, 0.0);
ML_Aggregate_Set_Threshold(thislevel_ag_, 0.0);
ML_Aggregate_Set_MaxCoarseSize(thislevel_ag_,1);
ML_Aggregate_Set_NullSpace(thislevel_ag_,numPDE,nullspdim,NULL,Mat->NumMyRows());
int thislevel_nlevel = ML_Gen_MGHierarchy_UsingAggregation(thislevel_ml_,0,
ML_INCREASING,thislevel_ag_);
if (thislevel_nlevel != 1)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: ML generated a local hierarchy of " << thislevel_nlevel << " on level " << level_ << "\n"
<< "**ERR**: this is supposed to be 1 Level only!\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// set the smoother
if (level_==0)
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,fsmoothertype,nsmooth_fine);
else if (level_ != nlevel_-1) // set the smoother from the input
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,smoothertype,nsmooth);
else // set the coarse solver from the input
Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,coarsesolvetype,nsmooth_coarse);
// create this level's preconditioner class
ML_Epetra::MultiLevelOperator* ml_tmp = new ML_Epetra::MultiLevelOperator(
thislevel_ml_,comm_,
Mat->OperatorDomainMap(),
Mat->OperatorRangeMap());
thislevel_prec_ = new ML_NOX::ML_Nox_ConstrainedMultiLevelOperator(ml_tmp,*coarseinterface_);
if (!thislevel_prec_)
{
cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
<< "**ERR**: thislevel_prec_==NULL on level " << level_ << "\n"
<< "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
}
// intensive test of this level's ML-smoother
#if 0
{
cout << "Test of smoother on level " << level_ << endl;
Epetra_Vector *out = new Epetra_Vector(Copy,*xthis_,0);
out->PutScalar(0.0);
cout << "Input\n";
xthis_->PutScalar(1.0);
Mat->Multiply(false,*xthis_,*out);
xthis_->PutScalar(3.0);
cout << "rhs\n";
cout << *out;
double norm = 0.0;
out->Norm1(&norm);
cout << "Norm of rhs = " << norm << endl;
thislevel_prec_->ApplyInverse(*out,*xthis_);
cout << "result after smoother\n";
cout << *xthis_;
delete out; out = 0;
}
if (level_==2) exit(0);
#endif
// ------------------------------------------------------------------------
// generate this level's coarse prepostoperator
if (level_==0)
coarseprepost_ = new ML_NOX::Ml_Nox_CoarsePrePostOperator(*coarseinterface_,
fineinterface_);
// ------------------------------------------------------------------------
// set up NOX on this level
// ------------------------------------------------------------------------
nlParams_ = new Teuchos::ParameterList();
Teuchos::ParameterList& printParams = nlParams_->sublist("Printing");
printParams.setParameter("MyPID", comm_.MyPID());
printParams.setParameter("Output Precision", 9);
printParams.setParameter("Output Processor", 0);
if (ml_printlevel_>9)
printParams.setParameter("Output Information",
NOX::Utils::OuterIteration +
//NOX::Utils::OuterIterationStatusTest +
//NOX::Utils::InnerIteration +
//NOX::Utils::Parameters +
//NOX::Utils::Details +
NOX::Utils::Warning);
else if (ml_printlevel_>8)
printParams.setParameter("Output Information",
NOX::Utils::Warning);
else
printParams.setParameter("Output Information",0);
if (level_==0)
nlParams_->sublist("Solver Options").setParameter("User Defined Pre/Post Operator", *coarseprepost_);
nlParams_->setParameter("Nonlinear Solver", "Line Search Based");
Teuchos::ParameterList& searchParams = nlParams_->sublist("Line Search");
Teuchos::ParameterList* lsParamsptr = 0;
if (isnlnCG_)
{
searchParams.setParameter("Method", "NonlinearCG");
Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
dirParams.setParameter("Method", "NonlinearCG");
Teuchos::ParameterList& nlcgParams = dirParams.sublist("Nonlinear CG");
nlcgParams.setParameter("Restart Frequency", 10);
nlcgParams.setParameter("Precondition", "On");
nlcgParams.setParameter("Orthogonalize", "Polak-Ribiere");
//nlcgParams.setParameter("Orthogonalize", "Fletcher-Reeves");
Teuchos::ParameterList& lsParams = nlcgParams.sublist("Linear Solver");
lsParams.setParameter("Aztec Solver", "CG");
lsParams.setParameter("Max Iterations", 1);
lsParams.setParameter("Tolerance", 1e-11);
lsParams.setParameter("Output Frequency", 0);
lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
lsParams.setParameter("Preconditioner","User Defined");
}
else // Newton's method using ML-preconditioned Aztec as linear solver
{
searchParams.setParameter("Method", "Full Step");
// Sublist for direction
Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
dirParams.setParameter("Method", "Newton");
Teuchos::ParameterList& newtonParams = dirParams.sublist("Newton");
newtonParams.setParameter("Forcing Term Method", "Constant");
//newtonParams.setParameter("Forcing Term Method", "Type 1");
//newtonParams.setParameter("Forcing Term Method", "Type 2");
newtonParams.setParameter("Forcing Term Minimum Tolerance", 1.0e-6);
newtonParams.setParameter("Forcing Term Maximum Tolerance", 0.1);
Teuchos::ParameterList& lsParams = newtonParams.sublist("Linear Solver");
lsParamsptr = &lsParams;
lsParams.setParameter("Aztec Solver", "CG");
lsParams.setParameter("Max Iterations", nitersCG_);
lsParams.setParameter("Tolerance", conv_normF_); // FIXME? is this correct?
if (ml_printlevel_>8)
lsParams.setParameter("Output Frequency", 50);
else
lsParams.setParameter("Output Frequency", 0);
lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
lsParams.setParameter("Preconditioner","User Defined");
}
// create the initial guess
initialGuess_ = new NOX::Epetra::Vector(*xthis_, NOX::DeepCopy, true);
// NOTE: do not delete xthis_, it's used and destroyed by initialGuess_
// create the necessary interfaces
NOX::EpetraNew::Interface::Preconditioner* iPrec = 0;
NOX::EpetraNew::Interface::Required* iReq = 0;
NOX::EpetraNew::Interface::Jacobian* iJac = 0;
if (isnlnCG_)
{
// create the matrixfree operator used in the nlnCG
thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
// create the necessary interfaces
iPrec = 0;
iReq = coarseinterface_;
iJac = thislevel_A_;
// create the linear system
thislevel_linSys_ = new ML_NOX::Ml_Nox_LinearSystem(
*iJac,*thislevel_A_,*iPrec,
coarseinterface_,*thislevel_prec_,
*xthis_,ismatrixfree_,level_,ml_printlevel_);
// create the group
group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*thislevel_linSys_);
}
else // Modified Newton's method
{
if (!broyden_)
{
// create the necessary interfaces
iPrec = this;
iReq = coarseinterface_;
iJac = this;
// create the initial guess vector
clone_ = new Epetra_Vector(*xthis_);
// create the linear system
azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
printParams,*lsParamsptr,
*iJac,*SmootherA_,*iPrec,
*thislevel_prec_,*clone_);
}
else
{
// create the initial guess vector
clone_ = new Epetra_Vector(*xthis_);
// create the necessary interfaces
iPrec = this;
iReq = coarseinterface_;
Broyd_ = new NOX::EpetraNew::BroydenOperator(*nlParams_,*clone_,
*SmootherA_,false);
// create the linear system
azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
printParams,*lsParamsptr,
*Broyd_,*SmootherA_,*iPrec,
*thislevel_prec_,*clone_);
}
// create the group
group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*azlinSys_);
}
// create convergence test
create_Nox_Convergencetest(conv_normF_,conv_nupdate_,conv_maxiter_);
// create the solver
solver_ = new NOX::Solver::Manager(*group_,*combo2_,*nlParams_);
return;
}
int main(int argc, char *argv[]){
ML *ml_object;
int i, N_grids = 3, N_levels;
double sol[129], rhs[129];
ML_Aggregate *agg_object;
ML_Operator *data;
ML_Krylov *kdata;
#ifdef ML_MPI
MPI_Init(&argc,&argv);
#endif
for (i = 0; i < 129; i++) sol[i] = 0.;
for (i = 0; i < 129; i++) rhs[i] = 2.;
ML_Create (&ml_object, N_grids);
ML_Init_Amatrix (ml_object, 0, 129, 129, NULL);
ML_Set_Amatrix_Getrow(ml_object, 0, Poisson_getrow, NULL, 129);
ML_Set_Amatrix_Matvec(ml_object, 0, Poisson_matvec);
ML_Set_PrintLevel(10);
ML_Aggregate_Create(&agg_object);
ML_Aggregate_Set_MaxCoarseSize(agg_object,1);
N_levels = ML_Gen_MGHierarchy_UsingAggregation(ml_object, 0,
ML_INCREASING, agg_object);
/******** Begin code to set a Jacobi smoother ******
ML_Gen_Smoother_Jacobi(ml_object, ML_ALL_LEVELS, ML_PRESMOOTHER, 1, ML_DEFAULT);
******** End code to set a Jacobi smoother ******/
/******** Begin code to set a user-defined smoother ******/
ML_Get_Amatrix(ml_object, 0, &data);
ML_Set_Smoother(ml_object, 0, ML_BOTH, data, user_smoothing,"mine");
ML_Get_Amatrix(ml_object, 1, &data);
ML_Set_Smoother(ml_object, 1, ML_BOTH, data, user_smoothing,"mine");
ML_Get_Amatrix(ml_object, 2, &data);
ML_Set_Smoother(ml_object, 2, ML_BOTH, data, user_smoothing,"mine");
ML_Gen_Solver (ml_object, ML_MGV, 0, N_levels-1);
/* This example uses an internal CG solver within ML */
/* ML has limited Krylov methods support. It is intended */
/* that ML be used with another package that supplies */
/* more sophisticated Krylov solver options (such as those */
/* found in the Trilinos or Aztec packages. */
kdata = ML_Krylov_Create(ml_object->comm);
ML_Krylov_Set_PrintFreq( kdata, 1 );
ML_Krylov_Set_Method(kdata, ML_CG);
ML_Krylov_Set_Amatrix(kdata, &(ml_object->Amat[0]));
ML_Krylov_Set_PreconFunc(kdata, ML_MGVSolve_Wrapper);
ML_Krylov_Set_Precon(kdata, ml_object);
ML_Krylov_Set_Tolerance(kdata, 1.e-7);
ML_Krylov_Solve(kdata, 129, rhs, sol);
ML_Krylov_Destroy( &kdata );
ML_Aggregate_Destroy(&agg_object);
ML_Destroy(&ml_object);
/******** End code to set a user-defined smoother ******/
printf("answer is %e %e %e %e %e\n",sol[0],sol[1],sol[2],sol[3],sol[4]);
#ifdef ML_MPI
MPI_Finalize();
#endif
exit(EXIT_SUCCESS);
}
