Exemple #1
0
// ================================================ ====== ==== ==== == = 
int ML_Epetra::FaceMatrixFreePreconditioner::NodeAggregate(ML_Aggregate_Struct *&MLAggr,ML_Operator *&P,ML_Operator* TMT_ML,int &NumAggregates){
  /* Pull Teuchos Options */
  string CoarsenType = List_.get("aggregation: type", "Uncoupled");
  double Threshold   = List_.get("aggregation: threshold", 0.0);  
  int    NodesPerAggr = List_.get("aggregation: nodes per aggregate", 
                                  ML_Aggregate_Get_OptimalNumberOfNodesPerAggregate());

  string PrintMsg_ = "FMFP (Level 0): ";

  ML_Aggregate_Create(&MLAggr);
  ML_Aggregate_Set_MaxLevels(MLAggr, 2);
  ML_Aggregate_Set_StartLevel(MLAggr, 0);
  ML_Aggregate_Set_Threshold(MLAggr, Threshold);
  ML_Aggregate_Set_MaxCoarseSize(MLAggr,1);
  MLAggr->cur_level = 0;
  ML_Aggregate_Set_Reuse(MLAggr); 
  MLAggr->keep_agg_information = 1;  
  P = ML_Operator_Create(ml_comm_);
  
  /* Process Teuchos Options */
  if (CoarsenType == "Uncoupled")
    ML_Aggregate_Set_CoarsenScheme_Uncoupled(MLAggr);
  else if (CoarsenType == "Uncoupled-MIS"){
    ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr);
  }
  else if (CoarsenType == "METIS"){
    ML_Aggregate_Set_CoarsenScheme_METIS(MLAggr);
    ML_Aggregate_Set_NodesPerAggr(0, MLAggr, 0, NodesPerAggr);
  }/*end if*/
  else {
    if(!Comm_->MyPID()) printf("FMFP: Unsupported (1,1) block aggregation type(%s), resetting to uncoupled-mis\n",CoarsenType.c_str());
    ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr);
  }

  /* Aggregate Nodes */
  int printlevel=ML_Get_PrintLevel();
  ML_Set_PrintLevel(10);
  NumAggregates = ML_Aggregate_Coarsen(MLAggr, TMT_ML, &P, ml_comm_);
  ML_Set_PrintLevel(printlevel);

  if (NumAggregates == 0){
    cerr << "Found 0 aggregates, perhaps the problem is too small." << endl;
    ML_CHK_ERR(-2);
  }/*end if*/
  else if(very_verbose_) printf("[%d] FMFP: %d aggregates created invec_leng=%d\n",Comm_->MyPID(),NumAggregates,P->invec_leng);

  int globalAggs;
  Comm_->SumAll(&NumAggregates,&globalAggs,1);
  if( verbose_ && !Comm_->MyPID()) {
    std::cout << PrintMsg_ << "Aggregation threshold = " << Threshold << std::endl;
    std::cout << PrintMsg_ << "Global aggregates = " << globalAggs << std::endl;
    //ML_Aggregate_Print_Complexity(MLAggr);
  }


  if(P==0) {fprintf(stderr,"%s","ERROR: No tentative prolongator found\n");ML_CHK_ERR(-5);}
  return 0;
}  
int main(int argc, char *argv[])
{
  int    Nnodes=16*16;              /* Total number of nodes in the problem.*/
                                    /* 'Nnodes' must be a perfect square.   */
  int    MaxMgLevels=6;             /* Maximum number of Multigrid Levels   */
  int    Nits_per_presmooth=1;      /* # of pre & post smoothings per level */
  double tolerance = 1.0e-8;        /* At convergence:                      */
                                    /*   ||r_k||_2 < tolerance ||r_0||_2    */
  int smoothPe_flag = ML_YES;       /* ML_YES: smooth tentative prolongator */
                                    /* ML_NO: don't smooth prolongator      */

  /***************************************************************************/
  /* Select Hiptmair relaxation subsmoothers for the nodal and edge problems */
  /* Choices include                                                         */
  /*   1) ML_Gen_Smoother_SymGaussSeidel: this corresponds to a processor    */
  /*      local version of symmetric Gauss-Seidel/SOR. The number of sweeps  */
  /*      can be set via either 'edge_its' or 'nodal_its'. The damping can   */
  /*      be set via 'edge_omega' or 'nodal_omega'. When set to ML_DDEFAULT, */
  /*      the damping is set to '1' on one processor. On multiple processors */
  /*      a lower damping value is set. This is needed to converge processor */
  /*      local SOR.                                                         */
  /*   2) ML_Gen_Smoother_Cheby: this corresponds to polynomial relaxation.    */
  /*      The degree of the polynomial is set via 'edge_its' or 'nodal_its'. */
  /*      If the degree is '-1', Marian Brezina's MLS polynomial is chosen.  */
  /*      Otherwise, a Chebyshev polynomial is used over high frequencies    */
  /*      [ lambda_max/alpha , lambda_max]. Lambda_max is computed. 'alpha'  */
  /*      is hardwired in this example to correspond to twice the ratio of   */
  /*      unknowns in the fine and coarse meshes.                            */
  /*                                                                         */
  /* Using 'hiptmair_type' (see comments below) it is also possible to choose*/
  /* when edge and nodal problems are relaxed within the Hiptmair smoother.  */
  /***************************************************************************/

  void  *edge_smoother=(void *)     /* Edge relaxation:                     */
               ML_Gen_Smoother_Cheby; /*   ML_Gen_Smoother_Cheby            */
                                    /*     ML_Gen_Smoother_SymGaussSeidel   */
  void *nodal_smoother=(void *)     /* Nodal relaxation                     */
               ML_Gen_Smoother_Cheby;/*     ML_Gen_Smoother_Cheby           */
                                    /*     ML_Gen_Smoother_SymGaussSeidel   */

  int  edge_its = 3;                /* Iterations or polynomial degree for  */
  int  nodal_its = 3;               /* edge/nodal subsmoothers.             */
  double nodal_omega = ML_DDEFAULT, /* SOR damping parameter for noda/edge  */
         edge_omega  = ML_DDEFAULT; /* subsmoothers (see comments above).   */
  int   hiptmair_type=HALF_HIPTMAIR;/* FULL_HIPTMAIR: each invokation       */
                                    /*     smoothes on edges, then nodes,   */
                                    /*     and then once again on edges.    */
                                    /* HALF_HIPTMAIR: each pre-invokation   */
                                    /*     smoothes on edges, then nodes.   */
                                    /*     Each post-invokation smoothes    */
                                    /*     on nodes then edges. .           */


  ML_Operator  *Tmat, *Tmat_trans, **Tmat_array, **Tmat_trans_array;
  ML           *ml_edges, *ml_nodes;
  ML_Aggregate *ag;
  int          Nfine_edge, Ncoarse_edge, Nfine_node, Ncoarse_node, Nlevels;
  int          level, coarsest_level, itmp;
  double       edge_coarsening_rate, node_coarsening_rate, *rhs, *xxx;
  void         **edge_args, **nodal_args;
  struct       user_partition Edge_Partition = {NULL, NULL,0,0}, 
                                Node_Partition = {NULL, NULL,0,0};
  struct Tmat_data Tmat_data;
int i, Ntotal;
 ML_Comm *comm;

  /* See Aztec User's Guide for information on these variables */

#ifdef AZTEC
  AZ_MATRIX    *Ke_mat, *Kn_mat;
  AZ_PRECOND   *Pmat = NULL;
  int          proc_config[AZ_PROC_SIZE], options[AZ_OPTIONS_SIZE];
  double       params[AZ_PARAMS_SIZE], status[AZ_STATUS_SIZE];
#endif


  /* get processor information (proc id & # of procs) and set ML's printlevel. */

#ifdef ML_MPI
  MPI_Init(&argc,&argv);
#endif
#ifdef AZTEC
  AZ_set_proc_config(proc_config, COMMUNICATOR);
#endif
  ML_Set_PrintLevel(10);   /* set ML's output level: 0 gives least output */

  /* Set the # of global nodes/edges and partition both the edges and the */
  /* nodes over the processors. NOTE: I believe we assume that if an edge */
  /* is assigned to a processor at least one of its nodes must be also    */
  /* assigned to that processor.                                          */

  Node_Partition.Nglobal = Nnodes;
  Edge_Partition.Nglobal = Node_Partition.Nglobal*2;
  Node_Partition.type = NODE;
  Edge_Partition.type = EDGE;
#define perxodic
#ifdef periodic
Node_Partition.Nglobal += 2; 
#endif
  partition_edges(&Edge_Partition);
  partition_nodes(&Node_Partition);
xxx = (double *) ML_allocate((Edge_Partition.Nlocal+100)*sizeof(double)); 
rhs = (double *) ML_allocate((Edge_Partition.Nlocal+100)*sizeof(double)); 
 for (i = 0; i < Edge_Partition.Nlocal + 100; i++) xxx[i] = -1.;
 for (i = 0; i < Edge_Partition.Nlocal; i++) xxx[i] = (double) 
        Edge_Partition.my_global_ids[i];

update_ghost_edges(xxx, (void *) &Edge_Partition);


  /* Create an empty multigrid hierarchy and set the 'MaxMGLevels-1'th   */
  /* level discretization within this hierarchy to the ML matrix         */
  /* representing Ke (Maxwell edge discretization).                      */

  ML_Create(&ml_edges, MaxMgLevels);
#ifdef AZTEC
  /* Build Ke as an Aztec matrix. Use built-in function AZ_ML_Set_Amat() */
  /* to convert to an ML matrix and put in hierarchy.                    */

  Ke_mat = user_Ke_build(&Edge_Partition);
  AZ_ML_Set_Amat(ml_edges, MaxMgLevels-1, Edge_Partition.Nlocal,
      		 Edge_Partition.Nlocal, Ke_mat, proc_config);
#else
  /* Build Ke directly as an ML matrix.                                  */

  ML_Init_Amatrix      (ml_edges, MaxMgLevels-1, Edge_Partition.Nlocal,
			Edge_Partition.Nlocal, &Edge_Partition);

  Ntotal = Edge_Partition.Nlocal;
  if (Edge_Partition.nprocs == 2) Ntotal += Edge_Partition.Nghost;
  ML_Set_Amatrix_Getrow(ml_edges, MaxMgLevels-1,  Ke_getrow, update_ghost_edges, Ntotal);
  ML_Set_Amatrix_Matvec(ml_edges, MaxMgLevels-1,  Ke_matvec);

#endif



  /* Build an Aztec matrix representing an auxiliary nodal PDE problem.  */
  /* This should be a variable coefficient Poisson problem (with unknowns*/
  /* at the nodes). The coefficients should be chosen to reflect the     */
  /* conductivity of the original edge problems.                         */
  /* Create an empty multigrid hierarchy. Convert the Aztec matrix to an */
  /* ML matrix and put it in the 'MaxMGLevels-1' level of the hierarchy. */
  /* Note it is possible to multiply T'*T for get this matrix though this*/
  /* will not incorporate material properties.                           */

  ML_Create(&ml_nodes, MaxMgLevels);

#ifdef AZTEC
  Kn_mat = user_Kn_build( &Node_Partition);
  AZ_ML_Set_Amat(ml_nodes, MaxMgLevels-1, Node_Partition.Nlocal, 
		 Node_Partition.Nlocal, Kn_mat, proc_config);
#else
  ML_Init_Amatrix      (ml_nodes, MaxMgLevels-1 , Node_Partition.Nlocal,
			Node_Partition.Nlocal, &Node_Partition);
  Ntotal = Node_Partition.Nlocal;
  if (Node_Partition.nprocs == 2) Ntotal += Node_Partition.Nghost;
  ML_Set_Amatrix_Getrow(ml_nodes, MaxMgLevels-1,  Kn_getrow, update_ghost_nodes, Ntotal);
#endif

  /* Build an ML matrix representing the null space of the PDE problem. */
  /* This should be a discrete gradient (nodes to edges).               */

#ifdef AZTEC
    Tmat = user_T_build (&Edge_Partition, &Node_Partition, 
  		   &(ml_nodes->Amat[MaxMgLevels-1]));
#else
    Tmat = ML_Operator_Create(ml_nodes->comm);
    Tmat_data.edge = &Edge_Partition;
    Tmat_data.node = &Node_Partition;
    Tmat_data.Kn   = &(ml_nodes->Amat[MaxMgLevels-1]);

    ML_Operator_Set_ApplyFuncData( Tmat,	Node_Partition.Nlocal,
				   Edge_Partition.Nlocal, ML_EMPTY, (void *) &Tmat_data, 
				   Edge_Partition.Nlocal, NULL, 0);
    ML_Operator_Set_Getrow( Tmat, ML_INTERNAL, Edge_Partition.Nlocal,Tmat_getrow);
    ML_Operator_Set_ApplyFunc(Tmat, ML_INTERNAL, Tmat_matvec);
  ML_Comm_Create( &comm);

  ML_CommInfoOP_Generate( &(Tmat->getrow->pre_comm), update_ghost_nodes, 
			  &Node_Partition,comm, Tmat->invec_leng, 
			  Node_Partition.Nghost);
#endif


  /********************************************************************/
  /* Set some ML parameters.                                          */
  /*------------------------------------------------------------------*/
	
  ML_Set_ResidualOutputFrequency(ml_edges, 1);
  ML_Set_Tolerance(ml_edges, 1.0e-8);
  ML_Aggregate_Create( &ag );
  ML_Aggregate_Set_CoarsenScheme_Uncoupled(ag);
  ML_Aggregate_Set_DampingFactor(ag, 0.0); /* must use 0 for maxwell */
  ML_Aggregate_Set_MaxCoarseSize(ag, 30);
  ML_Aggregate_Set_Threshold(ag, 0.0);


  /********************************************************************/
  /*                      Set up Tmat_trans                           */
  /*------------------------------------------------------------------*/

  Tmat_trans = ML_Operator_Create(ml_edges->comm);
  ML_Operator_Transpose_byrow(Tmat, Tmat_trans);


  Nlevels=ML_Gen_MGHierarchy_UsingReitzinger(ml_edges, &ml_nodes,MaxMgLevels-1,
					     ML_DECREASING,ag,Tmat,Tmat_trans, 
					     &Tmat_array,&Tmat_trans_array, 
					     smoothPe_flag, 1.5);

  /* Set the Hiptmair subsmoothers */

  if (nodal_smoother == (void *) ML_Gen_Smoother_SymGaussSeidel) {
    nodal_args = ML_Smoother_Arglist_Create(2);
    ML_Smoother_Arglist_Set(nodal_args, 0, &nodal_its);
    ML_Smoother_Arglist_Set(nodal_args, 1, &nodal_omega);
  }
  if (edge_smoother == (void *) ML_Gen_Smoother_SymGaussSeidel) {
    edge_args = ML_Smoother_Arglist_Create(2);
    ML_Smoother_Arglist_Set(edge_args, 0, &edge_its);
    ML_Smoother_Arglist_Set(edge_args, 1, &edge_omega);
  }
  if (nodal_smoother == (void *) ML_Gen_Smoother_Cheby) {
    nodal_args = ML_Smoother_Arglist_Create(2);
    ML_Smoother_Arglist_Set(nodal_args, 0, &nodal_its);
    Nfine_node = Tmat_array[MaxMgLevels-1]->invec_leng;
    Nfine_node = ML_gsum_int(Nfine_node, ml_edges->comm);
  }
  if (edge_smoother == (void *) ML_Gen_Smoother_Cheby) {
    edge_args = ML_Smoother_Arglist_Create(2);
    ML_Smoother_Arglist_Set(edge_args, 0, &edge_its);
    Nfine_edge = Tmat_array[MaxMgLevels-1]->outvec_leng;
    Nfine_edge = ML_gsum_int(Nfine_edge, ml_edges->comm);
  }

  /****************************************************
  * Set up smoothers for all levels but the coarsest. *
  ****************************************************/
  coarsest_level = MaxMgLevels - Nlevels;

  for (level = MaxMgLevels-1; level > coarsest_level; level--)
    {
      if (edge_smoother == (void *) ML_Gen_Smoother_Cheby) {
	Ncoarse_edge = Tmat_array[level-1]->outvec_leng;
	Ncoarse_edge = ML_gsum_int(Ncoarse_edge, ml_edges->comm);
	edge_coarsening_rate =  2.*((double) Nfine_edge)/ ((double) Ncoarse_edge);
	ML_Smoother_Arglist_Set(edge_args, 1, &edge_coarsening_rate);
	Nfine_edge = Ncoarse_edge;
      }
      if (nodal_smoother == (void *) ML_Gen_Smoother_Cheby) {
	Ncoarse_node = Tmat_array[level-1]->invec_leng;
	Ncoarse_node = ML_gsum_int(Ncoarse_node, ml_edges->comm);
	node_coarsening_rate =  2.*((double) Nfine_node)/ ((double) Ncoarse_node);
	ML_Smoother_Arglist_Set(nodal_args, 1, &node_coarsening_rate);
	Nfine_node = Ncoarse_node;
      }
      ML_Gen_Smoother_Hiptmair(ml_edges, level, ML_BOTH, Nits_per_presmooth,
			       Tmat_array, Tmat_trans_array, NULL, edge_smoother,
			       edge_args, nodal_smoother,nodal_args, hiptmair_type);
    }

  /*******************************************
  * Set up coarsest level smoother
  *******************************************/

  if (edge_smoother == (void *) ML_Gen_Smoother_Cheby) {
    edge_coarsening_rate = (double) Nfine_edge;
    ML_Smoother_Arglist_Set(edge_args, 1, &edge_coarsening_rate);
  }
  if (nodal_smoother == (void *) ML_Gen_Smoother_Cheby) {
    node_coarsening_rate = (double) Nfine_node;
    ML_Smoother_Arglist_Set(nodal_args,1,&node_coarsening_rate);
  }
  ML_Gen_CoarseSolverSuperLU( ml_edges, coarsest_level);
  

  /* Must be called before invoking the preconditioner */
  ML_Gen_Solver(ml_edges, ML_MGV, MaxMgLevels-1, coarsest_level); 



  /* Set the initial guess and the right hand side. Invoke solver */	

  xxx = (double *) ML_allocate(Edge_Partition.Nlocal*sizeof(double)); 
  ML_random_vec(xxx, Edge_Partition.Nlocal, ml_edges->comm);
  rhs = (double *) ML_allocate(Edge_Partition.Nlocal*sizeof(double)); 
  ML_random_vec(rhs, Edge_Partition.Nlocal, ml_edges->comm);

#ifdef AZTEC
  /* Choose the Aztec solver and criteria. Also tell Aztec that */
  /* ML will be supplying the preconditioner.                   */

  AZ_defaults(options, params);
  options[AZ_solver]   = AZ_fixed_pt;
  options[AZ_solver]   = AZ_gmres;
  options[AZ_kspace]   = 80;
  params[AZ_tol]       = tolerance;
  AZ_set_ML_preconditioner(&Pmat, Ke_mat, ml_edges, options); 
  options[AZ_conv] = AZ_noscaled;
  AZ_iterate(xxx, rhs, options, params, status, proc_config, Ke_mat, Pmat, NULL);
#else
  ML_Iterate(ml_edges, xxx, rhs);
#endif


  /* clean up. */

  ML_Smoother_Arglist_Delete(&nodal_args);
  ML_Smoother_Arglist_Delete(&edge_args);
  ML_Aggregate_Destroy(&ag);
  ML_Destroy(&ml_edges);
  ML_Destroy(&ml_nodes);
#ifdef AZTEC
  AZ_free((void *) Ke_mat->data_org);
  AZ_free((void *) Ke_mat->val);
  AZ_free((void *) Ke_mat->bindx);
  if (Ke_mat  != NULL) AZ_matrix_destroy(&Ke_mat);
  if (Pmat  != NULL) AZ_precond_destroy(&Pmat);
  if (Kn_mat != NULL) AZ_matrix_destroy(&Kn_mat);
#endif
  free(xxx);
  free(rhs);
  ML_Operator_Destroy(&Tmat);
  ML_Operator_Destroy(&Tmat_trans);
  ML_MGHierarchy_ReitzingerDestroy(MaxMgLevels-2, &Tmat_array, &Tmat_trans_array);

#ifdef ML_MPI
  MPI_Finalize();
#endif
		
  return 0;
		
}
int main(int argc, char *argv[])
{
  int num_PDE_eqns=1, N_levels=3, nsmooth=2;

  int leng, level, N_grid_pts, coarsest_level;
  int leng1,leng2;
  /* See Aztec User's Guide for more information on the */
  /* variables that follow.                             */

  int    proc_config[AZ_PROC_SIZE], options[AZ_OPTIONS_SIZE];
  double params[AZ_PARAMS_SIZE], status[AZ_STATUS_SIZE];

  /* data structure for matrix corresponding to the fine grid */

  double *val = NULL, *xxx, *rhs, solve_time, setup_time, start_time;
  AZ_MATRIX *Amat;
  AZ_PRECOND *Pmat = NULL;
  ML *ml;
  FILE *fp;
  int i, j, Nrigid, *garbage, nblocks=0, *blocks = NULL, *block_pde=NULL;
  struct AZ_SCALING *scaling;
  ML_Aggregate *ag;
  double *mode, *rigid=NULL, alpha; 
  char filename[80];
  int    one = 1;
  int    proc,nprocs;
  char pathfilename[100];

#ifdef ML_MPI
  MPI_Init(&argc,&argv);
  /* get number of processors and the name of this processor */
  AZ_set_proc_config(proc_config, MPI_COMM_WORLD);
  proc   = proc_config[AZ_node];
  nprocs = proc_config[AZ_N_procs];
#else
  AZ_set_proc_config(proc_config, AZ_NOT_MPI);
  proc   = 0;
  nprocs = 1;
#endif

   if (proc_config[AZ_node] == 0) {
      sprintf(pathfilename,"%s/inputfile",argv[1]);
      ML_Reader_ReadInput(pathfilename, &context);
   }
   else context = (struct reader_context *) ML_allocate(sizeof(struct reader_context));
   AZ_broadcast((char *) context,  sizeof(struct reader_context), proc_config,
                AZ_PACK);
   AZ_broadcast((char *) NULL        ,   0          , proc_config, AZ_SEND);

   N_levels = context->N_levels;
   printf("N_levels %d\n",N_levels);
   nsmooth   = context->nsmooth;
   num_PDE_eqns = context->N_dofPerNode;
   printf("num_PDE_eqns %d\n",num_PDE_eqns);

   ML_Set_PrintLevel(context->output_level);

  /* read in the number of matrix equations */
  leng = 0;
  if (proc_config[AZ_node] == 0) {
        sprintf(pathfilename,"%s/data_matrix.txt",argv[1]);
        fp=fopen(pathfilename,"r");
     if (fp==NULL) {
        printf("**ERR** couldn't open file data_matrix.txt\n");
        exit(1);
     }
        fscanf(fp,"%d",&leng);
     fclose(fp);
  }
  leng = AZ_gsum_int(leng, proc_config);

  N_grid_pts=leng/num_PDE_eqns;


  /* initialize the list of global indices. NOTE: the list of global */
  /* indices must be in ascending order so that subsequent calls to  */
  /* AZ_find_index() will function properly. */
#if 0
  if (proc_config[AZ_N_procs] == 1) i = AZ_linear;
  else i = AZ_file;
#endif
  i = AZ_linear;

  /* cannot use AZ_input_update for variable blocks (forgot why, but debugged through it)*/
  /* make a linear distribution of the matrix       */
  /* if the linear distribution does not align with the blocks, */
  /* this is corrected in ML_AZ_Reader_ReadVariableBlocks */
  leng1 = leng/nprocs;
  leng2 = leng-leng1*nprocs;
  if (proc >= leng2)
  {
     leng2 += (proc*leng1);
  }
  else
  {
     leng1++;
     leng2 = proc*leng1;
  }
  N_update = leng1;
  update = (int*)AZ_allocate((N_update+1)*sizeof(int));
  if (update==NULL)
  {
      (void) fprintf (stderr, "Not enough space to allocate 'update'\n");
      fflush(stderr); exit(EXIT_FAILURE);
  }
  for (i=0; i<N_update; i++) update[i] = i+leng2;
  
#if 0 /* debug */
  printf("proc %d N_update %d\n",proc_config[AZ_node],N_update);
  fflush(stdout);                   
#endif
  sprintf(pathfilename,"%s/data_vblocks.txt",argv[1]);
  ML_AZ_Reader_ReadVariableBlocks(pathfilename,&nblocks,&blocks,&block_pde,
                                  &N_update,&update,proc_config);
#if 0 /* debug */
  printf("proc %d N_update %d\n",proc_config[AZ_node],N_update);
  fflush(stdout);                   
#endif

  sprintf(pathfilename,"%s/data_matrix.txt",argv[1]);
  AZ_input_msr_matrix(pathfilename,update, &val, &bindx, N_update, proc_config);

  /* This code is to fix things up so that we are sure we have   */ 
  /* all blocks (including the ghost nodes) the same size.       */
  /* not sure, whether this is a good idea with variable blocks  */
  /* the examples inpufiles (see top of this file) don't need it */
  /* anyway                                                      */
  /*
  AZ_block_MSR(&bindx, &val, N_update, num_PDE_eqns, update);
  */
  AZ_transform_norowreordering(proc_config, &external, bindx, val,  update, &update_index,
	       &extern_index, &data_org, N_update, 0, 0, 0, &cpntr,
	       AZ_MSR_MATRIX);
	
  Amat = AZ_matrix_create( leng );

  AZ_set_MSR(Amat, bindx, val, data_org, 0, NULL, AZ_LOCAL);

  Amat->matrix_type  = data_org[AZ_matrix_type];
	
  data_org[AZ_N_rows]  = data_org[AZ_N_internal] + data_org[AZ_N_border];
			
  start_time = AZ_second();

  options[AZ_scaling] = AZ_none;

  ML_Create(&ml, N_levels);
			
			
  /* set up discretization matrix and matrix vector function */
  AZ_ML_Set_Amat(ml, 0, N_update, N_update, Amat, proc_config);

  ML_Set_ResidualOutputFrequency(ml, context->output);
  ML_Set_Tolerance(ml, context->tol);
  ML_Aggregate_Create( &ag );
  if (ML_strcmp(context->agg_coarsen_scheme,"Mis") == 0) {
     ML_Aggregate_Set_CoarsenScheme_MIS(ag);
  }
  else if (ML_strcmp(context->agg_coarsen_scheme,"Uncoupled") == 0) {
     ML_Aggregate_Set_CoarsenScheme_Uncoupled(ag);
  }
  else if (ML_strcmp(context->agg_coarsen_scheme,"Coupled") == 0) {
     ML_Aggregate_Set_CoarsenScheme_Coupled(ag);
  }
  else if (ML_strcmp(context->agg_coarsen_scheme,"Metis") == 0) {
     ML_Aggregate_Set_CoarsenScheme_METIS(ag);
     for (i=0; i<N_levels; i++)
        ML_Aggregate_Set_NodesPerAggr(ml,ag,i,9);
  }
  else if (ML_strcmp(context->agg_coarsen_scheme,"VBMetis") == 0) {
     /* when no blocks read, use standard metis assuming constant block sizes */
     if (!blocks) 
        ML_Aggregate_Set_CoarsenScheme_METIS(ag);
     else {
        ML_Aggregate_Set_CoarsenScheme_VBMETIS(ag);
        ML_Aggregate_Set_Vblocks_CoarsenScheme_VBMETIS(ag,0,N_levels,nblocks,
                                                       blocks,block_pde,N_update);
     }
     for (i=0; i<N_levels; i++)
        ML_Aggregate_Set_NodesPerAggr(ml,ag,i,9);
  }
  else {
     printf("**ERR** ML: Unknown aggregation scheme %s\n",context->agg_coarsen_scheme);
     exit(-1);
  }
  ML_Aggregate_Set_DampingFactor(ag, context->agg_damping);
  ML_Aggregate_Set_MaxCoarseSize( ag, context->maxcoarsesize);
  ML_Aggregate_Set_Threshold(ag, context->agg_thresh);

  if (ML_strcmp(context->agg_spectral_norm,"Calc") == 0) {
     ML_Set_SpectralNormScheme_Calc(ml);
  }
  else if (ML_strcmp(context->agg_spectral_norm,"Anorm") == 0) {
     ML_Set_SpectralNormScheme_Anorm(ml);
  }
  else {
     printf("**WRN** ML: Unknown spectral norm scheme %s\n",context->agg_spectral_norm);
  }

  /* read in the rigid body modes */

   Nrigid = 0;
   if (proc_config[AZ_node] == 0) {
      sprintf(filename,"data_nullsp%d.txt",Nrigid);
      sprintf(pathfilename,"%s/%s",argv[1],filename);
      while( (fp = fopen(pathfilename,"r")) != NULL) {
          fclose(fp);
          Nrigid++;
          sprintf(filename,"data_nullsp%d.txt",Nrigid);
          sprintf(pathfilename,"%s/%s",argv[1],filename);
      }
    }
    Nrigid = AZ_gsum_int(Nrigid,proc_config);

    if (Nrigid != 0) {
       rigid = (double *) ML_allocate( sizeof(double)*Nrigid*(N_update+1) );
       if (rigid == NULL) {
          printf("Error: Not enough space for rigid body modes\n");
       }
    }

   /* Set rhs */
   sprintf(pathfilename,"%s/data_rhs.txt",argv[1]);
   fp = fopen(pathfilename,"r");
   if (fp == NULL) {
      rhs=(double *)ML_allocate(leng*sizeof(double));
      if (proc_config[AZ_node] == 0) printf("taking linear vector for rhs\n");
      for (i = 0; i < N_update; i++) rhs[i] = (double) update[i];
   }
   else {
      fclose(fp);
      if (proc_config[AZ_node] == 0) printf("reading rhs from a file\n");
      AZ_input_msr_matrix(pathfilename, update, &rhs, &garbage, N_update, 
                          proc_config);
   }
   AZ_reorder_vec(rhs, data_org, update_index, NULL);

   for (i = 0; i < Nrigid; i++) {
      sprintf(filename,"data_nullsp%d.txt",i);
      sprintf(pathfilename,"%s/%s",argv[1],filename);
      AZ_input_msr_matrix(pathfilename, update, &mode, &garbage, N_update, 
                          proc_config);
      AZ_reorder_vec(mode, data_org, update_index, NULL);

#if 0 /* test the given rigid body mode, output-vector should be ~0 */
       Amat->matvec(mode, rigid, Amat, proc_config);
       for (j = 0; j < N_update; j++) printf("this is %d %e\n",j,rigid[j]);
#endif

    for (j = 0; j < i; j++) {
       alpha = -AZ_gdot(N_update, mode, &(rigid[j*N_update]), proc_config)/
                  AZ_gdot(N_update, &(rigid[j*N_update]), &(rigid[j*N_update]), 
                               proc_config);
       DAXPY_F77(&N_update, &alpha,  &(rigid[j*N_update]),  &one, mode, &one);
    }
   
    /* rhs orthogonalization */

    alpha = -AZ_gdot(N_update, mode, rhs, proc_config)/
                    AZ_gdot(N_update, mode, mode, proc_config);
    DAXPY_F77(&N_update, &alpha,  mode,  &one, rhs, &one);

    for (j = 0; j < N_update; j++) rigid[i*N_update+j] = mode[j];
    free(mode);
    free(garbage);
  }

  for (j = 0; j < Nrigid; j++) {
     alpha = -AZ_gdot(N_update, rhs, &(rigid[j*N_update]), proc_config)/
              AZ_gdot(N_update, &(rigid[j*N_update]), &(rigid[j*N_update]), 
                      proc_config);
     DAXPY_F77(&N_update, &alpha,  &(rigid[j*N_update]),  &one, rhs, &one);
  }

#if 0 /* for testing the default nullsp */
  ML_Aggregate_Set_NullSpace(ag, num_PDE_eqns, 6, NULL, N_update);
#else
  if (Nrigid != 0) {
     ML_Aggregate_Set_NullSpace(ag, num_PDE_eqns, Nrigid, rigid, N_update);
  }
#endif
  if (rigid) ML_free(rigid);

  ag->keep_agg_information = 1;
  coarsest_level = ML_Gen_MGHierarchy_UsingAggregation(ml, 0, 
                                            ML_INCREASING, ag);
  coarsest_level--;                                            

  if ( proc_config[AZ_node] == 0 )
	printf("Coarse level = %d \n", coarsest_level);
	
#if 0
  /* set up smoothers */
  if (!blocks)
     blocks = (int *) ML_allocate(sizeof(int)*N_update);
#endif

  for (level = 0; level < coarsest_level; level++) {

      num_PDE_eqns = ml->Amat[level].num_PDEs;
		
     /*  Sparse approximate inverse smoother that acutally does both */
     /*  pre and post smoothing.                                     */

     if (ML_strcmp(context->smoother,"Parasails") == 0) {
        ML_Gen_Smoother_ParaSails(ml , level, ML_PRESMOOTHER, nsmooth, 
                                parasails_sym, parasails_thresh, 
                                parasails_nlevels, parasails_filter,
                                (int) parasails_loadbal, parasails_factorized);
     }

     /* This is the symmetric Gauss-Seidel smoothing that we usually use. */
     /* In parallel, it is not a true Gauss-Seidel in that each processor */
     /* does a Gauss-Seidel on its local submatrix independent of the     */
     /* other processors.                                                 */

     else if (ML_strcmp(context->smoother,"GaussSeidel") == 0) {
       ML_Gen_Smoother_GaussSeidel(ml , level, ML_BOTH, nsmooth,1.);
     }
     else if (ML_strcmp(context->smoother,"SymGaussSeidel") == 0) {
       ML_Gen_Smoother_SymGaussSeidel(ml , level, ML_BOTH, nsmooth,1.);
     }
     else if (ML_strcmp(context->smoother,"Poly") == 0) {
       ML_Gen_Smoother_Cheby(ml, level, ML_BOTH, 30., nsmooth);
     }
     else if (ML_strcmp(context->smoother,"BlockGaussSeidel") == 0) {
       ML_Gen_Smoother_BlockGaussSeidel(ml , level, ML_BOTH, nsmooth,1.,
					 num_PDE_eqns);
     }
     else if (ML_strcmp(context->smoother,"VBSymGaussSeidel") == 0) {
         if (blocks)    ML_free(blocks);
         if (block_pde) ML_free(block_pde);
         blocks    = NULL;
         block_pde = NULL;
         nblocks   = 0;
         ML_Aggregate_Get_Vblocks_CoarsenScheme_VBMETIS(ag,level,N_levels,&nblocks,
                                                        &blocks,&block_pde);
         if (blocks==NULL) ML_Gen_Blocks_Aggregates(ag, level, &nblocks, &blocks);
         ML_Gen_Smoother_VBlockSymGaussSeidel(ml , level, ML_BOTH, nsmooth,1.,
                                              nblocks, blocks);
     }

     /* This is a true Gauss Seidel in parallel. This seems to work for  */
     /* elasticity problems.  However, I don't believe that this is very */
     /* efficient in parallel.                                           */       
     /*
      nblocks = ml->Amat[level].invec_leng;
      for (i =0; i < nblocks; i++) blocks[i] = i;
      ML_Gen_Smoother_VBlockSymGaussSeidelSequential(ml , level, ML_PRESMOOTHER,
                                                  nsmooth, 1., nblocks, blocks);
      ML_Gen_Smoother_VBlockSymGaussSeidelSequential(ml, level, ML_POSTSMOOTHER,
                                                  nsmooth, 1., nblocks, blocks);
     */

     /* Jacobi Smoothing                                                 */

     else if (ML_strcmp(context->smoother,"Jacobi") == 0) {
        ML_Gen_Smoother_Jacobi(ml , level, ML_PRESMOOTHER, nsmooth,.4);
        ML_Gen_Smoother_Jacobi(ml , level, ML_POSTSMOOTHER, nsmooth,.4);
     }

     /*  This does a block Gauss-Seidel (not true GS in parallel)        */
     /*  where each processor has 'nblocks' blocks.                      */
     /* */

     else if (ML_strcmp(context->smoother,"Metis") == 0) {
         if (blocks)    ML_free(blocks);
         if (block_pde) ML_free(block_pde);
         nblocks = 250;
         ML_Gen_Blocks_Metis(ml, level, &nblocks, &blocks);
         ML_Gen_Smoother_VBlockSymGaussSeidel(ml , level, ML_BOTH, nsmooth,1.,
                                        nblocks, blocks);
     }
     else {
         printf("unknown smoother %s\n",context->smoother);
         exit(1);
     }
   }
	
   /* set coarse level solver */
   nsmooth   = context->coarse_its;
   /*  Sparse approximate inverse smoother that acutally does both */
   /*  pre and post smoothing.                                     */

   if (ML_strcmp(context->coarse_solve,"Parasails") == 0) {
        ML_Gen_Smoother_ParaSails(ml , coarsest_level, ML_PRESMOOTHER, nsmooth, 
                                parasails_sym, parasails_thresh, 
                                parasails_nlevels, parasails_filter,
                                (int) parasails_loadbal, parasails_factorized);
   }

   else if (ML_strcmp(context->coarse_solve,"GaussSeidel") == 0) {
       ML_Gen_Smoother_GaussSeidel(ml , coarsest_level, ML_BOTH, nsmooth,1.);
   }
   else if (ML_strcmp(context->coarse_solve,"Poly") == 0) {
     ML_Gen_Smoother_Cheby(ml, coarsest_level, ML_BOTH, 30., nsmooth);
   }
   else if (ML_strcmp(context->coarse_solve,"SymGaussSeidel") == 0) {
       ML_Gen_Smoother_SymGaussSeidel(ml , coarsest_level, ML_BOTH, nsmooth,1.);
   }
   else if (ML_strcmp(context->coarse_solve,"BlockGaussSeidel") == 0) {
       ML_Gen_Smoother_BlockGaussSeidel(ml, coarsest_level, ML_BOTH, nsmooth,1.,
					num_PDE_eqns);
   }
   else if (ML_strcmp(context->coarse_solve,"Aggregate") == 0) {
         if (blocks)    ML_free(blocks);
         if (block_pde) ML_free(block_pde);
         ML_Gen_Blocks_Aggregates(ag, coarsest_level, &nblocks, &blocks);
         ML_Gen_Smoother_VBlockSymGaussSeidel(ml , coarsest_level, ML_BOTH, 
                                        nsmooth,1., nblocks, blocks);
   }
   else if (ML_strcmp(context->coarse_solve,"Jacobi") == 0) {
        ML_Gen_Smoother_Jacobi(ml , coarsest_level, ML_BOTH, nsmooth,.5);
   }
   else if (ML_strcmp(context->coarse_solve,"Metis") == 0) {
         if (blocks)    ML_free(blocks);
         if (block_pde) ML_free(block_pde);
         nblocks = 250;
         ML_Gen_Blocks_Metis(ml, coarsest_level, &nblocks, &blocks);
         ML_Gen_Smoother_VBlockSymGaussSeidel(ml , coarsest_level, ML_BOTH, 
                                              nsmooth,1., nblocks, blocks);
   }
   else if (ML_strcmp(context->coarse_solve,"SuperLU") == 0) {
      ML_Gen_CoarseSolverSuperLU( ml, coarsest_level);
   }
   else if (ML_strcmp(context->coarse_solve,"Amesos") == 0) {
      ML_Gen_Smoother_Amesos(ml,coarsest_level,ML_AMESOS_KLU,-1, 0.0);
   }
   else {
         printf("unknown coarse grid solver %s\n",context->coarse_solve);
         exit(1);
   }
		
   ML_Gen_Solver(ml, ML_MGV, 0, coarsest_level); 

   AZ_defaults(options, params);
	
   if (ML_strcmp(context->krylov,"Cg") == 0) {
      options[AZ_solver]   = AZ_cg;
   }
   else if (ML_strcmp(context->krylov,"Bicgstab") == 0) {
      options[AZ_solver]   = AZ_bicgstab;
   }
   else if (ML_strcmp(context->krylov,"Tfqmr") == 0) {
      options[AZ_solver]   = AZ_tfqmr;
   }
   else if (ML_strcmp(context->krylov,"Gmres") == 0) {
      options[AZ_solver]   = AZ_gmres;
   }
   else {
      printf("unknown krylov method %s\n",context->krylov);
   }
   if (blocks)            ML_free(blocks);
   if (block_pde)         ML_free(block_pde);
   options[AZ_scaling]  = AZ_none;
   options[AZ_precond]  = AZ_user_precond;
   options[AZ_conv]     = AZ_r0;
   options[AZ_output]   = 1;
   options[AZ_max_iter] = context->max_outer_its;
   options[AZ_poly_ord] = 5;
   options[AZ_kspace]   = 130;
   params[AZ_tol]       = context->tol;
   options[AZ_output]   = context->output;
   ML_free(context);
	
   AZ_set_ML_preconditioner(&Pmat, Amat, ml, options); 
   setup_time = AZ_second() - start_time;
	
   xxx = (double *) malloc( leng*sizeof(double));

   for (iii = 0; iii < leng; iii++) xxx[iii] = 0.0; 
	

   /* Set x */
   /*
   there is no initguess supplied with these examples for the moment....
   */
   fp = fopen("initguessfile","r");
   if (fp != NULL) {
      fclose(fp);
      if (proc_config[AZ_node]== 0) printf("reading initial guess from file\n");
      AZ_input_msr_matrix("data_initguess.txt", update, &xxx, &garbage, N_update, 
                          proc_config);

      options[AZ_conv] = AZ_expected_values;
   }
   else if (proc_config[AZ_node]== 0) printf("taking 0 initial guess \n");

   AZ_reorder_vec(xxx, data_org, update_index, NULL);

   /* if Dirichlet BC ... put the answer in */

   for (i = 0; i < data_org[AZ_N_internal]+data_org[AZ_N_border]; i++) {
      if ( (val[i] > .99999999) && (val[i] < 1.0000001))
         xxx[i] = rhs[i];      
   }

   fp = fopen("AZ_no_multilevel.dat","r");
   scaling = AZ_scaling_create();
   start_time = AZ_second();
   if (fp != NULL) {
      fclose(fp);
      options[AZ_precond] = AZ_none;
      options[AZ_scaling] = AZ_sym_diag;
      options[AZ_ignore_scaling] = AZ_TRUE;

      options[AZ_keep_info] = 1;
      AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling); 

/*
      options[AZ_pre_calc] = AZ_reuse;
      options[AZ_conv] = AZ_expected_values;
      if (proc_config[AZ_node] == 0) 
              printf("\n-------- Second solve with improved convergence test -----\n");
      AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling); 
      if (proc_config[AZ_node] == 0) 
              printf("\n-------- Third solve with improved convergence test -----\n");
      AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, NULL, scaling); 
*/
   }
   else {
      options[AZ_keep_info] = 1;
      AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling); 
      options[AZ_pre_calc] = AZ_reuse;
      options[AZ_conv] = AZ_expected_values;
/*
      if (proc_config[AZ_node] == 0) 
              printf("\n-------- Second solve with improved convergence test -----\n");
      AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling); 
      if (proc_config[AZ_node] == 0) 
              printf("\n-------- Third solve with improved convergence test -----\n");
      AZ_iterate(xxx, rhs, options, params, status, proc_config, Amat, Pmat, scaling); 
*/
   }
   solve_time = AZ_second() - start_time;

   if (proc_config[AZ_node] == 0) 
      printf("Solve time = %e, MG Setup time = %e\n", solve_time, setup_time);

   if (proc_config[AZ_node] == 0) 
     printf("Printing out a few entries of the solution ...\n");

   for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
     if (update[j] == 7) {printf("solution(gid = %d) = %10.4e\n",
			      update[j],xxx[update_index[j]]); fflush(stdout);}
   j = AZ_gsum_int(7, proc_config); /* sync processors */
   for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
     if (update[j] == 23) {printf("solution(gid = %d) = %10.4e\n",
			      update[j],xxx[update_index[j]]); fflush(stdout);}
   j = AZ_gsum_int(7, proc_config); /* sync processors */
   for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
     if (update[j] == 47) {printf("solution(gid = %d) = %10.4e\n",
			      update[j],xxx[update_index[j]]); fflush(stdout);}
   j = AZ_gsum_int(7, proc_config); /* sync processors */
   for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
     if (update[j] == 101) {printf("solution(gid = %d) = %10.4e\n",
			      update[j],xxx[update_index[j]]); fflush(stdout);}
   j = AZ_gsum_int(7, proc_config); /* sync processors */
   for (j=0;j<Amat->data_org[AZ_N_internal]+ Amat->data_org[AZ_N_border];j++)
     if (update[j] == 171) {printf("solution(gid = %d) = %10.4e\n",
			      update[j],xxx[update_index[j]]); fflush(stdout);}

   ML_Aggregate_Destroy(&ag);
   ML_Destroy(&ml);
   AZ_free((void *) Amat->data_org);
   AZ_free((void *) Amat->val);
   AZ_free((void *) Amat->bindx);
   AZ_free((void *) update);
   AZ_free((void *) external);
   AZ_free((void *) extern_index);
   AZ_free((void *) update_index);
   AZ_scaling_destroy(&scaling);
   if (Amat  != NULL) AZ_matrix_destroy(&Amat);
   if (Pmat  != NULL) AZ_precond_destroy(&Pmat);
   free(xxx);
   free(rhs);

#ifdef ML_MPI
  MPI_Finalize();
#endif
	
  return 0;
	
}
// ================================================ ====== ==== ==== == =
int ML_Epetra::RefMaxwell_Aggregate_Nodes(const Epetra_CrsMatrix & A, Teuchos::ParameterList & List, ML_Comm * ml_comm, std::string PrintMsg,
					  ML_Aggregate_Struct *& MLAggr,ML_Operator *&P, int &NumAggregates){

  /* Output level */
  bool verbose, very_verbose;
  int OutputLevel = List.get("ML output", -47);
  if(OutputLevel == -47) OutputLevel = List.get("output", 1);
  if(OutputLevel>=15) very_verbose=verbose=true;
  if(OutputLevel > 5) {very_verbose=false;verbose=true;}
  else very_verbose=verbose=false;

  /* Wrap A in a ML_Operator */
  ML_Operator* A_ML = ML_Operator_Create(ml_comm);
  ML_Operator_WrapEpetraCrsMatrix(const_cast<Epetra_CrsMatrix*>(&A),A_ML);

 /* Pull Teuchos Options */
  std::string CoarsenType  = List.get("aggregation: type", "Uncoupled");
  double Threshold    = List.get("aggregation: threshold", 0.0);
  int    NodesPerAggr = List.get("aggregation: nodes per aggregate",
                                  ML_Aggregate_Get_OptimalNumberOfNodesPerAggregate());
  bool UseAux         = List.get("aggregation: aux: enable",false);
  double AuxThreshold = List.get("aggregation: aux: threshold",0.0);
  int  MaxAuxLevels   = List.get("aggregation: aux: max levels",10);


  ML_Aggregate_Create(&MLAggr);
  ML_Aggregate_Set_MaxLevels(MLAggr, 2);
  ML_Aggregate_Set_StartLevel(MLAggr, 0);
  ML_Aggregate_Set_Threshold(MLAggr, Threshold);
  ML_Aggregate_Set_MaxCoarseSize(MLAggr,1);
  MLAggr->cur_level = 0;
  ML_Aggregate_Set_Reuse(MLAggr);
  MLAggr->keep_agg_information = 1;
  P = ML_Operator_Create(ml_comm);

  /* Process Teuchos Options */
  if (CoarsenType == "Uncoupled")
    ML_Aggregate_Set_CoarsenScheme_Uncoupled(MLAggr);
  else if (CoarsenType == "Uncoupled-MIS"){
    ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr);
  }
  else if (CoarsenType == "METIS"){
    ML_Aggregate_Set_CoarsenScheme_METIS(MLAggr);
    ML_Aggregate_Set_NodesPerAggr(0, MLAggr, 0, NodesPerAggr);
  }/*end if*/
  else {
    if(!A.Comm().MyPID()) printf("%s Unsupported (1,1) block aggregation type(%s), resetting to uncoupled-mis\n",PrintMsg.c_str(),CoarsenType.c_str());
    ML_Aggregate_Set_CoarsenScheme_UncoupledMIS(MLAggr);
  }

  /* Setup Aux Data */
  if(UseAux) {
    A_ML->aux_data->enable=1;
    A_ML->aux_data->threshold=AuxThreshold;
    A_ML->aux_data->max_level=MaxAuxLevels;
    ML_Init_Aux(A_ML,List);
    if(verbose && !A.Comm().MyPID()) {
      printf("%s Using auxiliary matrix\n",PrintMsg.c_str());
      printf("%s aux threshold = %e\n",PrintMsg.c_str(),A_ML->aux_data->threshold);
    }
  }

  /* Aggregate Nodes */
  int printlevel=ML_Get_PrintLevel();
  if(verbose) ML_Set_PrintLevel(10);
  NumAggregates = ML_Aggregate_Coarsen(MLAggr,A_ML, &P, ml_comm);
  if(verbose) ML_Set_PrintLevel(printlevel);

  if (NumAggregates == 0){
    std::cerr << "Found 0 aggregates, perhaps the problem is too small." << std::endl;
    ML_CHK_ERR(-2);
  }/*end if*/
  else if(very_verbose) printf("[%d] %s %d aggregates created invec_leng=%d\n",A.Comm().MyPID(),PrintMsg.c_str(),NumAggregates,P->invec_leng);

  if(verbose){
    int globalAggs=0;
    A.Comm().SumAll(&NumAggregates,&globalAggs,1);
    if(!A.Comm().MyPID()) {
      printf("%s Aggregation threshold = %e\n",PrintMsg.c_str(),Threshold);
      printf("%s Global aggregates     = %d\n",PrintMsg.c_str(),globalAggs);

    }
  }

  /* Cleanup */
  ML_qr_fix_Destroy();
  if(UseAux) ML_Finalize_Aux(A_ML);
  ML_Operator_Destroy(&A_ML);

  return 0;
}
/*----------------------------------------------------------------------*
 |  Constructor (public)                                     m.gee 01/05|
 |  IMPORTANT:                                                          |
 |  No matter on which level we are here, the vector xfine is ALWAYS    |
 |  a fine grid vector here!                                            |
 |  this is the constructor for the ismatrixfree==false case
 *----------------------------------------------------------------------*/
ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel(
                          int level, int nlevel, int printlevel, ML* ml, 
                          ML_Aggregate* ag,Epetra_CrsMatrix** P, 
                          ML_NOX::Ml_Nox_Fineinterface& interface,
                          const Epetra_Comm& comm,  const Epetra_Vector& xfine, 
                          bool ismatrixfree, bool matfreelev0, bool isnlnCG,
                          int nitersCG, bool broyden, Epetra_CrsMatrix* Jac, 
                          string fsmoothertype, string smoothertype, 
                          string coarsesolvetype, 
                          int nsmooth_fine, int nsmooth, int nsmooth_coarse,  
                          double conv_normF, double conv_nupdate, 
                          int conv_maxiter,int numPDE, int nullspdim) 
: fineinterface_(interface),
  comm_(comm)
{
   level_            = level;        // this level
   nlevel_           = nlevel;       // number of total levels
   ml_printlevel_    = printlevel;   // printlevel
   ml_               = ml;           // the global ML object
   ag_               = ag;           // the global ML_Aggregate object
   thislevel_prec_   = 0;            // this level's linear preconditioner
   thislevel_ml_     = 0;            // this level's local ML object 
   thislevel_ag_     = 0;            // this level's local ML_Aggregate object
   coarseinterface_  = 0;            // this level's coarse interface
   coarseprepost_    = 0;
   xthis_            = 0;            // this level's current solution matching this level's map!!!!
   thislevel_A_      = 0;            // this level's NOX Matrixfree operator
   SmootherA_        = 0;            // this level's Epetra_CrsMatrix for thislevel_prec_
   ismatrixfree_     = ismatrixfree; // matrixfree flag
   conv_normF_       = conv_normF;   // NOX convergence test stuff
   conv_nupdate_     = conv_nupdate;
   conv_maxiter_     = conv_maxiter;
   absresid_         = 0;            
   nupdate_          = 0;
   fv_               = 0;
   maxiters_         = 0;
   combo1_           = 0;
   combo2_           = 0;
   thislevel_linSys_ = 0;            // this level's NOX linear system
   nlParams_         = 0;            // NOX parameters
   initialGuess_     = 0;            // NOX initial guess
   group_            = 0;            // NOX group
   solver_           = 0;            // NOX solver
   isnlnCG_          = isnlnCG;
   azlinSys_         = 0;
   clone_            = 0;
   nitersCG_         = nitersCG;
   broyden_          = broyden;
   Broyd_            = 0;

   if (ismatrixfree_==true)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: ismatrixfree_==true on level " << level_ << "\n"
           << "**ERR**: in constructor for ismatrixfree_==false - case\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
   
   // ------------------------------------------------------------------------
   // get the Jacobian of this level
   const Epetra_CrsGraph* graph = 0;
   // ------------------------------------------------------------------------
   if (level_==0)
   {
      graph = fineinterface_.getGraph();
      // On fine level this is the fineinterface's Jacobian
      if (matfreelev0==false)
         SmootherA_ = fineinterface_.getJacobian();
      else if (matfreelev0==true && Jac)
         SmootherA_ = Jac;
      else
      {
        cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
             << "**ERR**: something weired happened\n"
             << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
      }
   }
   // ------------------------------------------------------------------------
   else
   {
      // On coarse levels get Jacobian from hierarchy
      // Note: On levels>0 SmootherA_ is a real copy of the Jacobian
      int maxnnz=0;
      double cputime=0.0;
      ML_Operator2EpetraCrsMatrix(&(ml_->Amat[level_]), SmootherA_, maxnnz, 
                                  false, cputime);
      SmootherA_->OptimizeStorage();
      graph = &(SmootherA_->Graph());
   }
   
   // just to be save
   if (!SmootherA_ || !graph)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: Smoother==NULL on level " << level_ << "\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }

   // ------------------------------------------------------------------------
   // generate this level's coarse interface
   coarseinterface_ = new ML_NOX::Nox_CoarseProblem_Interface(
                                      fineinterface_,level_,ml_printlevel_,
                                      P,&(graph->RowMap()),nlevel_);  

   // ------------------------------------------------------------------------
   // generate this level's coarse prepostoperator
   if (level_==0)
     coarseprepost_ = new ML_NOX::Ml_Nox_CoarsePrePostOperator(*coarseinterface_,
                                                               fineinterface_);    
   
   // ------------------------------------------------------------------------
   // get the current solution to this level
   xthis_ = coarseinterface_->restrict_fine_to_this(xfine);
   

   // ------------------------------------------------------------------------
   // create this level's preconditioner
   // We use a 1-level ML-hierarchy for that
   ML_Aggregate_Create(&thislevel_ag_);
   ML_Create(&thislevel_ml_,1);
   
   // set the Jacobian on level 0 of the local ml
   EpetraMatrix2MLMatrix(thislevel_ml_,0,
                         (dynamic_cast<Epetra_RowMatrix*>(SmootherA_)));
   
   // construct a 1-level ML-hierarchy on this level as a smoother   
   ML_Set_PrintLevel(ml_printlevel_);  
   ML_Aggregate_Set_CoarsenScheme_Uncoupled(thislevel_ag_); 
   ML_Aggregate_Set_DampingFactor(thislevel_ag_, 0.0);
   ML_Aggregate_Set_Threshold(thislevel_ag_, 0.0);
   ML_Aggregate_Set_MaxCoarseSize(thislevel_ag_,1);
   ML_Aggregate_Set_NullSpace(thislevel_ag_,numPDE,nullspdim,NULL,
                              SmootherA_->NumMyRows());
   int thislevel_nlevel = ML_Gen_MGHierarchy_UsingAggregation(thislevel_ml_,0,
                                                   ML_INCREASING,thislevel_ag_);
   if (thislevel_nlevel != 1)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: ML generated a local hierarchy of " <<  thislevel_nlevel << " on level " << level_ << "\n" 
           << "**ERR**: this is supposed to be 1 Level only!\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
   
   // set the smoother
   if (level_==0)
      Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,fsmoothertype,nsmooth_fine);

   else if (level_ != nlevel_-1) // set the smoother from the input
      Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,smoothertype,nsmooth);

   else // set the coarse solver from the input
      Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,coarsesolvetype,nsmooth_coarse);
  
   // create this level's preconditioner class
   ML_Epetra::MultiLevelOperator* ml_tmp = new ML_Epetra::MultiLevelOperator(
                                                thislevel_ml_,comm_,
                                                SmootherA_->OperatorDomainMap(),
                                                SmootherA_->OperatorRangeMap());
   thislevel_prec_ = new ML_NOX::ML_Nox_ConstrainedMultiLevelOperator(ml_tmp,*coarseinterface_);
   
   if (!thislevel_prec_)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: thislevel_prec_==NULL on level " << level_ << "\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
                                                                 
   // ------------------------------------------------------------------------
   // set up NOX on this level   
   // ------------------------------------------------------------------------
   nlParams_ = new Teuchos::ParameterList();
   Teuchos::ParameterList& printParams = nlParams_->sublist("Printing");        
   printParams.setParameter("MyPID", comm_.MyPID()); 
   printParams.setParameter("Output Precision", 14);
   printParams.setParameter("Output Processor", 0);
   if (ml_printlevel_>9)
      printParams.setParameter("Output Information",
   	                       NOX::Utils::OuterIteration + 
			       NOX::Utils::Warning);
  else if (ml_printlevel_>8)
      printParams.setParameter("Output Information",
			       NOX::Utils::Warning);
  else
      printParams.setParameter("Output Information",0);

  if (level_==0)
    nlParams_->sublist("Solver Options").setParameter("User Defined Pre/Post Operator", *coarseprepost_);
  nlParams_->setParameter("Nonlinear Solver", "Line Search Based");         
  Teuchos::ParameterList& searchParams = nlParams_->sublist("Line Search");
  Teuchos::ParameterList* lsParamsptr  = 0;
  if (isnlnCG_)
  {
     searchParams.setParameter("Method", "NonlinearCG");
     Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction"); 
     dirParams.setParameter("Method", "NonlinearCG");
     Teuchos::ParameterList& nlcgParams = dirParams.sublist("Nonlinear CG");
     nlcgParams.setParameter("Restart Frequency", 10);                         
     nlcgParams.setParameter("Precondition", "On");
     nlcgParams.setParameter("Orthogonalize", "Polak-Ribiere");
     //nlcgParams.setParameter("Orthogonalize", "Fletcher-Reeves");

     Teuchos::ParameterList& lsParams = nlcgParams.sublist("Linear Solver");     
     lsParams.setParameter("Aztec Solver", "CG"); 
     lsParams.setParameter("Max Iterations", 1);  
     lsParams.setParameter("Tolerance", 1e-11);
     lsParams.setParameter("Output Frequency", 0);   
     lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");   
     lsParams.setParameter("Preconditioner","User Defined");
  }
  else // Newton's method using ML-preconditioned Aztec as linear solver
  {
     searchParams.setParameter("Method", "Full Step");
     // Sublist for direction
     Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
     dirParams.setParameter("Method", "Newton");
     Teuchos::ParameterList& newtonParams = dirParams.sublist("Newton");
     newtonParams.setParameter("Forcing Term Method", "Constant");
     //newtonParams.setParameter("Forcing Term Method", "Type 1");
     //newtonParams.setParameter("Forcing Term Method", "Type 2");
     newtonParams.setParameter("Forcing Term Minimum Tolerance", 1.0e-6);
     newtonParams.setParameter("Forcing Term Maximum Tolerance", 0.1);

     Teuchos::ParameterList& lsParams = newtonParams.sublist("Linear Solver");
     lsParamsptr = &lsParams;
     lsParams.setParameter("Size of Krylov Subspace", 100);
     lsParams.setParameter("Aztec Solver", "GMRES"); 
     lsParams.setParameter("Max Iterations", nitersCG_);  
     lsParams.setParameter("Tolerance", conv_normF_); // FIXME? is this correct?
     if (ml_printlevel_>8)
        lsParams.setParameter("Output Frequency", 50);   
     else
        lsParams.setParameter("Output Frequency", 0);   
     lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
     lsParams.setParameter("Preconditioner","User Defined");
  }

  // create the initial guess     
  initialGuess_ = new NOX::Epetra::Vector(*xthis_, NOX::DeepCopy, true);
  // NOTE: do not delete xthis_, it's used and destroyed by initialGuess_

   
  // create the necessary interfaces
  NOX::EpetraNew::Interface::Preconditioner* iPrec = 0; 
  NOX::EpetraNew::Interface::Required*       iReq  = 0;
  NOX::EpetraNew::Interface::Jacobian*       iJac  = 0;

  if (isnlnCG_)
  {
     // create the matrixfree operator used in the nlnCG
     thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
  
     // create the necessary interfaces
     iPrec = 0; 
     iReq  = coarseinterface_;
     iJac  = thislevel_A_;
  
     // create the linear system 
     thislevel_linSys_ = new ML_NOX::Ml_Nox_LinearSystem(
                                    *iJac,*thislevel_A_,*iPrec,
                                    coarseinterface_,*thislevel_prec_,
                                    *xthis_,ismatrixfree_,level_,ml_printlevel_);
     // create the group
     group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*thislevel_linSys_);
  }
  else // Modified Newton's method
  {
     if (!broyden_)
     {
       // create the necessary interfaces   
       iPrec = this; 
       iReq  = coarseinterface_;
       //iJac  = this;
       thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
     
       // create the initial guess vector
       //clone_  = new Epetra_Vector(*xthis_);

       // create the linear system 
       //azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
       //                                               printParams,*lsParamsptr,
       //                                               *iJac,*SmootherA_,*iPrec,
       //                                               *thislevel_prec_,*clone_);
       azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
                                                      printParams,*lsParamsptr,
                                                      *thislevel_A_,*thislevel_A_,*iPrec,
                                                      *thislevel_prec_,*xthis_);
     }
     else // use a Broyden update for the Jacobian
     {
       // create the initial guess vector
       //clone_  = new Epetra_Vector(*xthis_);

       // create the necessary interfaces   
       iPrec  = this; 
       iReq   = coarseinterface_;
       Broyd_ = new NOX::EpetraNew::BroydenOperator(*nlParams_,*xthis_,
                                                    *SmootherA_,false);
       
       // create the linear system 
       azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
                                                   printParams,*lsParamsptr,
                                                   *Broyd_,*SmootherA_,*iPrec,
                                                   *thislevel_prec_,*xthis_);
       
     }
     // create the group
     group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,
                                        *azlinSys_);
  }

  // create convergence test
  create_Nox_Convergencetest(conv_normF_,conv_nupdate_,conv_maxiter_);

  // create the solver
  solver_ = new NOX::Solver::Manager(*group_,*combo2_,*nlParams_);

  return;
}
/*----------------------------------------------------------------------*
 |  Constructor (public)                                     m.gee 01/05|
 |  IMPORTANT:                                                          |
 |  No matter on which level we are here, the vector xfine is ALWAYS    |
 |  a fine grid vector here!                                            |
 |  this is the constructor for the ismatrixfree==true case
 *----------------------------------------------------------------------*/
ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel(
                          int level, int nlevel, int printlevel, ML* ml, 
                          ML_Aggregate* ag,Epetra_CrsMatrix** P, 
                          ML_NOX::Ml_Nox_Fineinterface& interface,
                          const Epetra_Comm& comm,  const Epetra_Vector& xfine, 
                          bool ismatrixfree, bool isnlnCG, int nitersCG, bool broyden,
                          string fsmoothertype, string smoothertype, string coarsesolvetype, 
                          int nsmooth_fine, int nsmooth, int nsmooth_coarse,  
                          double conv_normF, 
                          double conv_nupdate, int conv_maxiter,
                          int numPDE, int nullspdim, Epetra_CrsMatrix* Mat,
                          ML_NOX::Nox_CoarseProblem_Interface* coarseinterface) 
: fineinterface_(interface),
  comm_(comm)
{
   level_            = level;           // this level
   nlevel_           = nlevel;          // number of total levels
   ml_printlevel_    = printlevel;      // printlevel
   ml_               = ml;              // the global ML object
   ag_               = ag;              // the global ML_Aggregate object
   thislevel_prec_   = 0;               // this level's linear preconditioner
   thislevel_ml_     = 0;               // this level's local ML object 
   thislevel_ag_     = 0;               // this level's local ML_Aggregate object
   coarseinterface_  = coarseinterface; // this level's coarse interface
   coarseprepost_    = 0;
   xthis_            = 0;               // this level's current solution matching this level's map!!!!
   thislevel_A_      = 0;               // this level's NOX Matrixfree operator
   SmootherA_        = 0;               // this level's Epetra_CrsMatrix for thislevel_prec_
   ismatrixfree_     = ismatrixfree;    // matrixfree flag
   conv_normF_       = conv_normF;      // NOX convergence test stuff
   conv_nupdate_     = conv_nupdate;
   conv_maxiter_     = conv_maxiter;
   absresid_         = 0;            
   nupdate_          = 0;
   fv_               = 0;
   maxiters_         = 0;
   combo1_           = 0;
   combo2_           = 0;
   thislevel_linSys_ = 0;               // this level's NOX linear system
   nlParams_         = 0;               // NOX parameters
   initialGuess_     = 0;               // NOX initial guess
   group_            = 0;               // NOX group
   solver_           = 0;               // NOX solver
   SmootherA_        = Mat;
   isnlnCG_          = isnlnCG;
   azlinSys_         = 0;
   clone_            = 0;
   nitersCG_         = nitersCG;
   broyden_          = broyden;
   Broyd_            = 0;

#if 0   
   if (isnlnCG_==false && 
      (fsmoothertype   == "Jacobi" || 
       smoothertype    == "Jacobi" || 
       coarsesolvetype == "Jacobi" ))
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: Modified Newton's method not supported for \n"
           << "**ERR**: ismatrixfree_==true && smoothertype == Jacobi-Smoother\n"
           << "**ERR**: because no full Jacobian exists!\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
#endif   
   if (ismatrixfree_==false)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: ismatrixfree_==false on level " << level_ << "\n"
           << "**ERR**: in constructor for ismatrixfree_==true - case\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
   if (!coarseinterface_)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: ptr to coarseinterface=NULL on level " << level_ << "\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
   if (!Mat)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: ptr to Matrix Mat=NULL on level " << level_ << "\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
   
   // ------------------------------------------------------------------------
   Mat->OptimizeStorage();

   // ------------------------------------------------------------------------
   // get the current solution to this level
   xthis_ = coarseinterface_->restrict_fine_to_this(xfine);

   // ------------------------------------------------------------------------
   // create this level's preconditioner
   // We use a 1-level ML-hierarchy for that
   ML_Aggregate_Create(&thislevel_ag_);
   ML_Create(&thislevel_ml_,1);
   
   // ------------------------------------------------------------------------
   // set the Jacobian on level 0 of the local ml
   EpetraMatrix2MLMatrix(thislevel_ml_,0,
                         (dynamic_cast<Epetra_RowMatrix*>(Mat)));
   
   // ------------------------------------------------------------------------
   // construct a 1-level ML-hierarchy on this level as a smoother   
   // ------------------------------------------------------------------------
   ML_Set_PrintLevel(ml_printlevel_);  
   ML_Aggregate_Set_CoarsenScheme_Uncoupled(thislevel_ag_); 
   ML_Aggregate_Set_DampingFactor(thislevel_ag_, 0.0);
   ML_Aggregate_Set_Threshold(thislevel_ag_, 0.0);
   ML_Aggregate_Set_MaxCoarseSize(thislevel_ag_,1);
   ML_Aggregate_Set_NullSpace(thislevel_ag_,numPDE,nullspdim,NULL,Mat->NumMyRows());
   int thislevel_nlevel = ML_Gen_MGHierarchy_UsingAggregation(thislevel_ml_,0,
                                                   ML_INCREASING,thislevel_ag_);
   if (thislevel_nlevel != 1)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: ML generated a local hierarchy of " <<  thislevel_nlevel << " on level " << level_ << "\n" 
           << "**ERR**: this is supposed to be 1 Level only!\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
   
   
   // set the smoother
   if (level_==0)
      Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,fsmoothertype,nsmooth_fine);

   else if (level_ != nlevel_-1) // set the smoother from the input
      Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,smoothertype,nsmooth);

   else // set the coarse solver from the input
      Set_Smoother(ml,ag,level_,nlevel,thislevel_ml_,thislevel_ag_,coarsesolvetype,nsmooth_coarse);
  
   // create this level's preconditioner class
   ML_Epetra::MultiLevelOperator* ml_tmp = new ML_Epetra::MultiLevelOperator(
                                                       thislevel_ml_,comm_,
                                                       Mat->OperatorDomainMap(),
                                                       Mat->OperatorRangeMap());
   thislevel_prec_ = new ML_NOX::ML_Nox_ConstrainedMultiLevelOperator(ml_tmp,*coarseinterface_);

   if (!thislevel_prec_)
   {
      cout << "**ERR**: ML_NOX::ML_Nox_NonlinearLevel::ML_Nox_NonlinearLevel:\n"
           << "**ERR**: thislevel_prec_==NULL on level " << level_ << "\n"
           << "**ERR**: file/line: " << __FILE__ << "/" << __LINE__ << "\n"; throw -1;
   }
                                                                 
   // intensive test of this level's ML-smoother
#if 0
   {
   cout << "Test of smoother on level " << level_ << endl;
   Epetra_Vector *out = new Epetra_Vector(Copy,*xthis_,0);
   out->PutScalar(0.0);
   cout << "Input\n";
   xthis_->PutScalar(1.0);
   Mat->Multiply(false,*xthis_,*out);
   xthis_->PutScalar(3.0);
   cout << "rhs\n";
   cout << *out;
   double norm = 0.0;
   out->Norm1(&norm);
   cout << "Norm of rhs = " << norm << endl;
   thislevel_prec_->ApplyInverse(*out,*xthis_);
   cout << "result after smoother\n";
   cout << *xthis_;
   delete out; out = 0;
   }
   if (level_==2) exit(0);
#endif   

   // ------------------------------------------------------------------------
   // generate this level's coarse prepostoperator
   if (level_==0)
      coarseprepost_ = new ML_NOX::Ml_Nox_CoarsePrePostOperator(*coarseinterface_,
                                                                fineinterface_);    

   // ------------------------------------------------------------------------
   // set up NOX on this level   
   // ------------------------------------------------------------------------
   nlParams_ = new Teuchos::ParameterList();
   Teuchos::ParameterList& printParams = nlParams_->sublist("Printing");        
   printParams.setParameter("MyPID", comm_.MyPID()); 
   printParams.setParameter("Output Precision", 9);
   printParams.setParameter("Output Processor", 0);
   if (ml_printlevel_>9)
      printParams.setParameter("Output Information",
   	                       NOX::Utils::OuterIteration + 
			       //NOX::Utils::OuterIterationStatusTest + 
			       //NOX::Utils::InnerIteration +
			       //NOX::Utils::Parameters + 
			       //NOX::Utils::Details + 
			       NOX::Utils::Warning);
  else if (ml_printlevel_>8)
      printParams.setParameter("Output Information",
			       NOX::Utils::Warning);
  else
      printParams.setParameter("Output Information",0);

  if (level_==0)
    nlParams_->sublist("Solver Options").setParameter("User Defined Pre/Post Operator", *coarseprepost_);
  nlParams_->setParameter("Nonlinear Solver", "Line Search Based");         
  Teuchos::ParameterList& searchParams = nlParams_->sublist("Line Search");
  Teuchos::ParameterList* lsParamsptr  = 0;
  if (isnlnCG_)
  {
     searchParams.setParameter("Method", "NonlinearCG");
     Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction"); 
     dirParams.setParameter("Method", "NonlinearCG");
     Teuchos::ParameterList& nlcgParams = dirParams.sublist("Nonlinear CG");
     nlcgParams.setParameter("Restart Frequency", 10);                         
     nlcgParams.setParameter("Precondition", "On");
     nlcgParams.setParameter("Orthogonalize", "Polak-Ribiere");
     //nlcgParams.setParameter("Orthogonalize", "Fletcher-Reeves");

     Teuchos::ParameterList& lsParams = nlcgParams.sublist("Linear Solver");     
     lsParams.setParameter("Aztec Solver", "CG"); 
     lsParams.setParameter("Max Iterations", 1);  
     lsParams.setParameter("Tolerance", 1e-11);
     lsParams.setParameter("Output Frequency", 0);   
     lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");   
     lsParams.setParameter("Preconditioner","User Defined");
  }
  else // Newton's method using ML-preconditioned Aztec as linear solver
  {
     searchParams.setParameter("Method", "Full Step");
     // Sublist for direction
     Teuchos::ParameterList& dirParams = nlParams_->sublist("Direction");
     dirParams.setParameter("Method", "Newton");
     Teuchos::ParameterList& newtonParams = dirParams.sublist("Newton");
     newtonParams.setParameter("Forcing Term Method", "Constant");
     //newtonParams.setParameter("Forcing Term Method", "Type 1");
     //newtonParams.setParameter("Forcing Term Method", "Type 2");
     newtonParams.setParameter("Forcing Term Minimum Tolerance", 1.0e-6);
     newtonParams.setParameter("Forcing Term Maximum Tolerance", 0.1);

     Teuchos::ParameterList& lsParams = newtonParams.sublist("Linear Solver");
     lsParamsptr = &lsParams;
     lsParams.setParameter("Aztec Solver", "CG"); 
     lsParams.setParameter("Max Iterations", nitersCG_);  
     lsParams.setParameter("Tolerance", conv_normF_); // FIXME? is this correct?
     if (ml_printlevel_>8)
        lsParams.setParameter("Output Frequency", 50);   
     else
        lsParams.setParameter("Output Frequency", 0);   
     lsParams.setParameter("Preconditioning", "User Supplied Preconditioner");
     lsParams.setParameter("Preconditioner","User Defined");
  }
   
  // create the initial guess   
  initialGuess_ = new NOX::Epetra::Vector(*xthis_, NOX::DeepCopy, true);
  // NOTE: do not delete xthis_, it's used and destroyed by initialGuess_

  // create the necessary interfaces
  NOX::EpetraNew::Interface::Preconditioner* iPrec = 0; 
  NOX::EpetraNew::Interface::Required*       iReq  = 0;
  NOX::EpetraNew::Interface::Jacobian*       iJac  = 0;

  if (isnlnCG_)
  {
     // create the matrixfree operator used in the nlnCG
     thislevel_A_ = new NOX::EpetraNew::MatrixFree(*coarseinterface_,*xthis_,false);
  
     // create the necessary interfaces
     iPrec = 0; 
     iReq  = coarseinterface_;
     iJac  = thislevel_A_;
  
     // create the linear system 
     thislevel_linSys_ = new ML_NOX::Ml_Nox_LinearSystem(
                                    *iJac,*thislevel_A_,*iPrec,
                                    coarseinterface_,*thislevel_prec_,
                                    *xthis_,ismatrixfree_,level_,ml_printlevel_);

     // create the group
     group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*thislevel_linSys_);
  }
  else // Modified Newton's method
  {
     if (!broyden_)
     {
       // create the necessary interfaces   
       iPrec = this; 
       iReq  = coarseinterface_;
       iJac  = this;
     
       // create the initial guess vector
       clone_  = new Epetra_Vector(*xthis_);

       // create the linear system 
       azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
                                                      printParams,*lsParamsptr,
                                                      *iJac,*SmootherA_,*iPrec,
                                                      *thislevel_prec_,*clone_);
     }
     else
     {
       // create the initial guess vector
       clone_  = new Epetra_Vector(*xthis_);

       // create the necessary interfaces   
       iPrec = this; 
       iReq  = coarseinterface_;
       Broyd_ = new NOX::EpetraNew::BroydenOperator(*nlParams_,*clone_,
                                                    *SmootherA_,false);
     
       // create the linear system 
       azlinSys_ = new NOX::EpetraNew::LinearSystemAztecOO(
                                                   printParams,*lsParamsptr,
                                                   *Broyd_,*SmootherA_,*iPrec,
                                                   *thislevel_prec_,*clone_);
     }
     // create the group
     group_ = new NOX::EpetraNew::Group(printParams,*iReq,*initialGuess_,*azlinSys_);
  }


  // create convergence test
  create_Nox_Convergencetest(conv_normF_,conv_nupdate_,conv_maxiter_);

  // create the solver
  solver_ = new NOX::Solver::Manager(*group_,*combo2_,*nlParams_);
  
  return;
}