Пример #1
0
void AZK_create_linsys_no_copy(double *xr, double *xi, double *br, double *bi,
                               int *options, double *params, int *proc_config,
                               AZ_MATRIX *Amat_real, AZ_MATRIX *Amat_imag,
                               double **x, double **b, AZ_MATRIX **Amat)
{
    /*
       Transforms a complex-valued system

             (Ar +i*Ai) * (xr + i*xi) = (br + i*bi)

       where double precision arrays hold the real and imaginary parts separately.

     Input arguments:
     ================

     xr,xi:         On input, contains the initial guess, real part in xr and
                    imaginary part in xi. On output contains the solution to
                    the linear system.

     br,bi:         Right hand side of linear system.

     Output arguments:
     =================

     x:             Komplex version of initial guess and solution.
     b:             Komplex version of RHS.
     Amat:          Komplex version of matrix stored as an AZ_MATRIX structure.

    */
    AZ_KOMPLEX *linsys_pass_data;
    int N_equations, N_blk_equations, N_real, N_external;
    int *data_org_real, *data_org_imag;
    int *komplex_to_real, *komplex_to_imag;
    int i;


    if (Amat_real->has_global_indices || Amat_imag->has_global_indices)
        AZ_perror("AZK_create_linsys_no_copy requires local indices");

    linsys_pass_data = (AZ_KOMPLEX *) AZ_allocate(sizeof(AZ_KOMPLEX));
    if (linsys_pass_data == NULL)
        AZ_perror("AZK_create_linsys_no_copy: Out of memory.");
    data_org_real = Amat_real->data_org;
    data_org_imag = Amat_imag->data_org;
    N_real = data_org_real[AZ_N_internal] + data_org_real[AZ_N_border];

    N_equations = 2 * N_real;
    N_blk_equations = N_equations;

    N_external = AZ_MAX(data_org_real[AZ_N_external], data_org_imag[AZ_N_external]);

    if (Amat_real->matrix_type == AZ_MSR_MATRIX) {
        Amat_real->data_org[AZ_N_int_blk] = Amat_real->data_org[AZ_N_internal];
        Amat_real->data_org[AZ_N_bord_blk] = Amat_real->data_org[AZ_N_border];
        N_blk_equations = N_equations;
    }

    else if (Amat_real->matrix_type == AZ_VBR_MATRIX)
    {
        N_blk_equations = data_org_real[AZ_N_int_blk] + data_org_real[AZ_N_bord_blk];
    }
    else if (Amat_real->matrix_type == AZ_USER_MATRIX) {
        Amat_real->data_org[AZ_N_int_blk] = Amat_real->data_org[AZ_N_internal];
        Amat_real->data_org[AZ_N_bord_blk] = Amat_real->data_org[AZ_N_border];
        N_blk_equations = N_equations;
    }
    else
        AZ_perror("AZK_create_linsys_no_copy: Unknown matrix type.");

    if (Amat_imag->matrix_type == AZ_MSR_MATRIX) {
        Amat_imag->data_org[AZ_N_int_blk] = Amat_imag->data_org[AZ_N_internal];
        Amat_imag->data_org[AZ_N_bord_blk] = Amat_imag->data_org[AZ_N_border];
    }

    else if (Amat_imag->matrix_type == AZ_USER_MATRIX) {
        Amat_imag->data_org[AZ_N_int_blk] = Amat_imag->data_org[AZ_N_internal];
        Amat_imag->data_org[AZ_N_bord_blk] = Amat_imag->data_org[AZ_N_border];
    }

    (*Amat) = AZ_create_matrix(N_equations, 0,
                               AZ_USER_MATRIX, N_blk_equations,
                               AZ_NOT_USING_AZTEC_MATVEC);

    /* Merge real and imaginary parts into K matrix order */
    komplex_to_real = (int *) AZ_allocate (N_real*sizeof(int));
    komplex_to_imag = (int *) AZ_allocate (N_real*sizeof(int));
    (*x) = (double *) AZ_allocate((N_equations+N_external)*sizeof(double));
    (*b) = (double *) AZ_allocate((N_equations+N_external)*sizeof(double));
    if ((*b) == NULL)
        AZ_perror("AZK_create_linsys_no_copy: Out of memory.");
    for (i=0; i <N_real; i++)
    {
        komplex_to_real[i] = 2*i;
        komplex_to_imag[i] = 2*i+1;
        (*x)[komplex_to_real[i]] = xr[i];
        (*x)[komplex_to_imag[i]] = xi[i];
        (*b)[komplex_to_real[i]] = br[i];
        (*b)[komplex_to_imag[i]] = bi[i];
    }
    linsys_pass_data->Amat_real = Amat_real;
    linsys_pass_data->Amat_imag = Amat_imag;
    linsys_pass_data->komplex_to_real = komplex_to_real;
    linsys_pass_data->komplex_to_imag = komplex_to_imag;

    linsys_pass_data->c11 = 1.0;
    linsys_pass_data->c12 = 0.0;
    linsys_pass_data->c21 = 0.0;
    linsys_pass_data->c22 = 1.0;
    linsys_pass_data->Form_of_Equations = AZK_Komplex_No_Copy;
    linsys_pass_data->From_Global_Indices = 0;

    (*Amat)->matvec  = AZK_matvec_no_copy;
    (*Amat)->aux_ptr = (void *)  linsys_pass_data;

    return;
}
Пример #2
0
void read_coo(char *data_file, int *proc_config,
	      int *N_global, int *n_nonzeros,
	      double **val, int **bindx,
	      double **x, double **b, double **xexact)
#undef DEBUG 
     /*  read ASCII data file:
	 line 1: N_global, number of entries (%d,%d)
	 line 2-...: i,j,real (%d, %d, %f)
     */

{
  FILE *data ;


  int i, n_entries, N_columns;
  int ii, jj ;
  int kk = 0;
  int max_ii = 0, max_jj = 0;
  int ione = 1;
  double value;
  double *cnt;
  int *pntr, *indx1, *pntr1;
  double *val1;

  int MAXBLOCKSIZE = 25;
  
  if(proc_config[AZ_node] == 0) 
    { 

 
      data = fopen(data_file,"r") ;
 
      fscanf(data, "%d %d %d", N_global, &N_columns, &n_entries) ;
      if (N_columns != *N_global)
	perror("Matrix dimensions must be the same");
      printf("Reading from file: %s\n",data_file);
      printf("Number of equations = %d\n",*N_global);
      printf("Number of entries   = %d\n",n_entries);
    

      *bindx = (int   *) calloc(n_entries+1,sizeof(int)) ;
      *val = (double *) calloc(n_entries+1,sizeof(double)) ;

      pntr1 = (int   *) calloc(n_entries+1,sizeof(int)) ;
      indx1 = (int   *) calloc(n_entries+1,sizeof(int)) ;
      val1 = (double *) calloc(n_entries+1,sizeof(double)) ;
  
      pntr = (int   *) calloc(n_entries+1,sizeof(int)) ;

      if ((pntr) == NULL) 
	perror("Error: Not enough space to create matrix");

      while(!feof(data))
	{
	  fscanf(data, "%d %d %lf", &ii, &jj, &value) ;
	  max_ii = AZ_MAX(max_ii,ii);
	  max_jj = AZ_MAX(max_jj,jj);
#ifdef DEBUG
	  printf("Entry %d, %d = %lf.\n",ii,jj,value);
#endif
	  (*bindx)[kk] = ii;
	  pntr[kk] = jj;
	  (*val)[kk] = value;
	  kk++;
	}
      *n_nonzeros = kk-1;
      *N_global = max_ii;
      if (max_ii != max_jj) perror("Error: Number of rows and columns not equal");

      printf("Number of nonzeros = %d\n",*n_nonzeros);

      /* Convert real part in the following way:
	 - Convert COO to CSR
	 - CSR to CSC
	 - CSC to CSR (columns are now in ascending order)
	 - CSR to MSR
      */
      coocsr_(N_global,n_nonzeros, *val, *bindx, pntr, val1, indx1, pntr1);

      csrcsc_(N_global,&ione,&ione,
	      val1,indx1,pntr1,
	      *val,*bindx,pntr);

      csrcsc_(N_global,&ione,&ione,
	      *val,*bindx,pntr,
	      val1,indx1,pntr1);

      csrmsr_(N_global,val1,indx1,pntr1,
	      *val,*bindx,
	      *val,*bindx);

      /* Finally, convert bindx vectors to zero base */

      for (i=0;i<*n_nonzeros+1;i++)
	(*bindx)[i] -= 1;

      *b = (double   *) calloc((*N_global)*MAXBLOCKSIZE,sizeof(double)) ;
      *x = (double   *) calloc((*N_global)*MAXBLOCKSIZE,sizeof(double)) ;

      if ((*x) == NULL) 
	perror("Error: Not enough space to create matrix");

  
      /* Set RHS to a random vector, initial guess to zero */
      for (i=0;i<*N_global;i++)
	{
	  (*b)[i] = drand48();
	  (*x)[i] = 0.0;
	}
  }
   
      /* Release unneeded space */

  free((void *) pntr);
  free((void *) val1);
  free((void *) indx1);
  free((void *) pntr1);
  
  
  /* end read_coo */
}
Пример #3
0
int update_solution(double** x, double** delta_x, int iter) {
/* Routine to update solution vector x using delta_x and 
 * to test for convergence of Newton's method.
 * 
 * Note: Modifications to Newton's method, including damping, have not yet
 *       been translated form previous update_solution.
 *       iter  value may be used in some damping methods
 */

  int iunk, ibox, inode,inodeG,ijk[3],go_update,idim,nnodes_loop;
  double updateNorm=0.0, temp,frac_min,frac;
  char *yo = "newupdate solution";

  if (Type_poly==WJDC3 && Grafted_Logical==TRUE) nnodes_loop=Nnodes_box_extra;
  else nnodes_loop=Nnodes_box;
     
   /* Certain unknowns - specifically densities and Gs in CMS DFT cannot be less than 0.
      Here we locate problems, and scale the entire update vector to prevent this from 
      happening. */
  frac_min=1.0;
  for (ibox=0; ibox<nnodes_loop; ibox++) { /* find minimum update fraction in entire domain */
    for (iunk=0; iunk<Nunk_per_node; iunk++){
      if ( (Unk2Phys[iunk]==G_CHAIN  && Pol_Sym[iunk-Phys2Unk_first[G_CHAIN]] == -1) ||
           (Unk2Phys[iunk]==DENSITY && (!(Type_poly==WTC) || (Pol_Sym_Seg[iunk-Phys2Unk_first[DENSITY]] ==-1) )) ){
         if(x[iunk][ibox]+delta_x[iunk][ibox]<0.0){
             frac = AZ_MIN(1.0,x[iunk][ibox]/(-delta_x[iunk][ibox]));
             frac = AZ_MAX(frac,NL_update_scalingParam);
         } 
        else{
             frac=1.0;
         }

         if (frac<frac_min) frac_min=frac;
      }
    }
  }

  frac_min=gmin_double(frac_min);
  if (Proc==0 && Iwrite_screen != SCREEN_NONE && Iwrite_screen != SCREEN_ERRORS_ONLY){
     if (Iwrite_screen == SCREEN_BASIC) printf("\tUpdate percent=%g ",frac_min*100);
     else printf("\tUpdate Frac = %g percent\n",frac_min*100);
  }

  for (ibox=0; ibox<nnodes_loop; ibox++) {

    /* Increment updateNorm only for owned nodes (inode=-1 for ghosts) */
    inode = B2L_node[ibox];
    if (inode != -1) {
      for (iunk=0; iunk<Nunk_per_node; iunk++) {
        temp =(frac_min*delta_x[iunk][ibox])/(NL_rel_tol*x[iunk][ibox] + NL_abs_tol);
        updateNorm +=  temp*temp;
      }
    }

  /* For some cases, we need to be able to keep the solution values at the boundaries constant
     and set equal to the values that are read in from a file.  Do not update the solution 
     vector at these points */


    inodeG=B2G_node[ibox]; 
    node_to_ijk(inodeG,ijk);
    go_update=TRUE;
    for (idim=0; idim<Ndim;idim++){
       if (  (ijk[idim]==0 && Type_bc[idim][0] == LAST_NODE_RESTART) ||
          (ijk[idim]==Nodes_x[idim]-1 && Type_bc[idim][1] == LAST_NODE_RESTART)) go_update=FALSE;
    }
   
    /* Update all solution componenets */
    if (go_update){
    for (iunk=0; iunk<Nunk_per_node; iunk++){

     /* if (x[iunk][ibox]+frac_min*delta_x[iunk][ibox] <0.0 && */
      if (x[iunk][ibox]+frac_min*delta_x[iunk][ibox] <1.e-99 && 
         (  (Unk2Phys[iunk]==DENSITY && (!(Type_poly==WTC) || (Pol_Sym_Seg[iunk-Phys2Unk_first[DENSITY]] ==-1) )) || 
            (Unk2Phys[iunk]==G_CHAIN && Pol_Sym[iunk-Phys2Unk_first[G_CHAIN]] == -1)                              || 
            Unk2Phys[iunk]==CMS_FIELD                                                                             || 
            Unk2Phys[iunk]==WJDC_FIELD                                                                            ||
            (Unk2Phys[iunk]==BONDWTC  && Pol_Sym[iunk-Phys2Unk_first[BONDWTC]] == -1 )                            || 
             Unk2Phys[iunk]==CAVWTC)                 
          ){

            x[iunk][ibox]=0.1*x[iunk][ibox];
      }
      else if ((iunk==Phys2Unk_first[HSRHOBAR] || iunk==(Phys2Unk_first[CAVWTC]+1)) && 
              x[iunk][ibox]+frac_min*delta_x[iunk][ibox] > 1.0){
              x[iunk][ibox]+=0.5*(1.0-x[iunk][ibox]);
      }
      else{
         x[iunk][ibox] += frac_min*delta_x[iunk][ibox];
      }
    }
    }
  }
  

  updateNorm = sqrt(gsum_double(updateNorm));

  if (Proc==0 && Iwrite_screen != SCREEN_NONE && Iwrite_screen != SCREEN_ERRORS_ONLY){
    if (Iwrite_screen == SCREEN_BASIC) printf("\tWeighted norm update vec =  %g\n", updateNorm);
    else    printf("\n\t\tWeighted norm of update vector =  %g\n", updateNorm);
  }


  if (updateNorm > 1.0) return(FALSE);
  else                  return(TRUE);

}
Пример #4
0
int main(int argc, char *argv[])
{
  char  global[]="global";
  char  local[]="local";

  int    proc_config[AZ_PROC_SIZE];/* Processor information.                */
  int    options[AZ_OPTIONS_SIZE]; /* Array used to select solver options.  */
  double params[AZ_PARAMS_SIZE];   /* User selected solver paramters.       */
  int    *data_org;
                                   /* Array to specify data layout          */
  double status[AZ_STATUS_SIZE];   /* Information returned from AZ_solve(). */
  int    *update;                  /* vector elements updated on this node. */
  int    *external;
                                   /* vector elements needed by this node.  */
  int    *update_index;
                                   /* ordering of update[] and external[]   */
  int    *extern_index;
                                   /* locally on this processor.            */
  int    *indx;   /* MSR format of real and imag parts */
  int    *bindx;
  int    *bpntr;
  int    *rpntr;
  int    *cpntr;
  AZ_MATRIX *Amat;
  AZ_PRECOND *Prec;
  double *val;
  double *x, *b, *xexact, *xsolve;
  int    n_nonzeros, n_blk_nonzeros;
  int    N_update;           /* # of block unknowns updated on this node    */
  int    N_local;
                                 /* Number scalar equations on this node */
  int    N_global, N_blk_global; /* Total number of equations */
  int    N_external, N_blk_eqns;

  double *val_msr;
  int *bindx_msr;
  
  double norm, d ;

  int matrix_type;

  int has_global_indices, option;
  int i, j, m, mp ;
  int ione = 1;

#ifdef TEST_SINGULAR
  double * xnull; /* will contain difference of given exact solution and computed solution*/
  double * Axnull; /* Product of A time xnull */
  double norm_Axnull;
#endif

#ifdef AZTEC_MPI
  double MPI_Wtime(void) ;
#endif
  double time ;
#ifdef AZTEC_MPI
  MPI_Init(&argc,&argv);
#endif

  /* get number of processors and the name of this processor */
 
#ifdef AZTEC_MPI
  AZ_set_proc_config(proc_config,MPI_COMM_WORLD);
#else
  AZ_set_proc_config(proc_config,0);
#endif

  printf("proc %d of %d is alive\n",
	 proc_config[AZ_node],proc_config[AZ_N_procs]) ;

#ifdef AZTEC_MPI
  MPI_Barrier(MPI_COMM_WORLD) ;
#endif

#ifdef VBRMATRIX
  if(argc != 3) 
    perror("error: enter name of data and partition file on command line") ; 
#else
  if(argc != 2) perror("error: enter name of data file on command line") ; 
#endif
  /* Set exact solution to NULL */
  xexact = NULL;

  /* Read matrix file and distribute among processors.  
     Returns with this processor's set of rows */ 

#ifdef VBRMATRIX
  read_hb(argv[1], proc_config, &N_global, &n_nonzeros, 
	  &val_msr,  &bindx_msr, &x, &b, &xexact);
  
  create_vbr(argv[2], proc_config, &N_global, &N_blk_global,
	     &n_nonzeros, &n_blk_nonzeros, &N_update, &update,
	     bindx_msr, val_msr, &val, &indx, 
	     &rpntr, &cpntr, &bpntr, &bindx);

  if(proc_config[AZ_node] == 0) 
    {
      free ((void *) val_msr);
      free ((void *) bindx_msr);
      free ((void *) cpntr);
    }
    matrix_type = AZ_VBR_MATRIX;

#ifdef AZTEC_MPI
  MPI_Barrier(MPI_COMM_WORLD) ;
#endif

  distrib_vbr_matrix( proc_config, N_global, N_blk_global, 
		      &n_nonzeros, &n_blk_nonzeros,
		      &N_update, &update, 
		      &val, &indx, &rpntr, &cpntr, &bpntr, &bindx, 
		      &x, &b, &xexact);

#else
    read_hb(argv[1], proc_config, &N_global, &n_nonzeros,
             &val,  &bindx, &x, &b, &xexact);

#ifdef AZTEC_MPI
  MPI_Barrier(MPI_COMM_WORLD) ;
#endif

  distrib_msr_matrix(proc_config, N_global, &n_nonzeros, &N_update,
		  &update, &val, &bindx, &x, &b, &xexact);

#ifdef DEBUG
  for (i = 0; i<N_update; i++)
    if (val[i] == 0.0 ) printf("Zero diagonal at row %d\n",i);
#endif
    matrix_type = AZ_MSR_MATRIX;
#endif
  /* convert matrix to a local distributed matrix */
    cpntr = NULL;
  AZ_transform(proc_config, &external, bindx, val, update,
	       &update_index, &extern_index, &data_org, 
	       N_update, indx, bpntr, rpntr, &cpntr,
               matrix_type);

  printf("Processor %d: Completed AZ_transform\n",proc_config[AZ_node]) ;
      has_global_indices = 0;
      option = AZ_LOCAL;

#ifdef VBRMATRIX
  N_local = rpntr[N_update];
#else
  N_local = N_update;
#endif

  Amat = AZ_matrix_create(N_local);

#ifdef VBRMATRIX
  AZ_set_VBR(Amat, rpntr, cpntr, bpntr, indx, bindx, val, data_org,
          N_update, update, option);
#else
  AZ_set_MSR(Amat, bindx, val, data_org, N_update, update, option);
#endif


  printf("proc %d Completed AZ_create_matrix\n",proc_config[AZ_node]) ;

#ifdef AZTEC_MPI
  MPI_Barrier(MPI_COMM_WORLD) ;
#endif

  /* initialize AZTEC options */
 
  AZ_defaults(options, params);
  options[AZ_solver]  = AZ_gmres;
  options[AZ_precond] = AZ_sym_GS; 
  options[AZ_poly_ord] = 1;
  options[AZ_graph_fill] = 1;
  params[AZ_rthresh] = 0.0E-7;
  params[AZ_athresh] = 0.0E-7;
  options[AZ_overlap] = 1;
 /*
  params[AZ_ilut_fill] = 2.0;
  params[AZ_drop] = 0.01;
  options[AZ_overlap] = 0;
  options[AZ_reorder] = 0;
  params[AZ_rthresh] = 1.0E-1;
  params[AZ_athresh] = 1.0E-1;
  options[AZ_precond] = AZ_dom_decomp ;
  options[AZ_subdomain_solve] = AZ_bilu_ifp;
  options[AZ_reorder] = 0;
  options[AZ_graph_fill] = 0;
  params[AZ_rthresh] = 1.0E-7;
  params[AZ_athresh] = 1.0E-7;
 options[AZ_poly_ord] = 1;
 options[AZ_precond] = AZ_Jacobi;
  params[AZ_omega] = 1.0;
  options[AZ_precond] = AZ_none ;

  options[AZ_poly_ord] = 1;
  options[AZ_precond] = AZ_Jacobi ;
  options[AZ_scaling] = AZ_sym_row_sum ;
  options[AZ_scaling] = AZ_sym_diag;


  options[AZ_conv] = AZ_noscaled;
  options[AZ_scaling] = AZ_Jacobi ;

  options[AZ_precond] = AZ_dom_decomp ;
  options[AZ_subdomain_solve] = AZ_icc ;
  options[AZ_subdomain_solve] = AZ_ilut ;
  params[AZ_omega] = 1.2;
  params[AZ_ilut_fill] = 2.0;
  params[AZ_drop] = 0.01;
  options[AZ_reorder] = 0;
  options[AZ_overlap] = 0;
  options[AZ_type_overlap] = AZ_symmetric;

  options[AZ_precond] = AZ_dom_decomp ;
  options[AZ_subdomain_solve] = AZ_bilu ;
  options[AZ_graph_fill] = 0;
  options[AZ_overlap] = 0;

  options[AZ_precond] = AZ_dom_decomp ;
  options[AZ_subdomain_solve] = AZ_bilu_ifp ;
  options[AZ_graph_fill] = 0;
  options[AZ_overlap] = 0;
  params[AZ_rthresh] = 1.0E-3;
  params[AZ_athresh] = 1.0E-3;

 options[AZ_poly_ord] = 1;
 options[AZ_precond] = AZ_Jacobi ; */


  options[AZ_kspace] = 600 ;

  options[AZ_max_iter] = 600 ;
  params[AZ_tol] = 1.0e-14;

#ifdef BGMRES
  options[AZ_gmres_blocksize] = 3;
  options[AZ_gmres_num_rhs] = 1;
#endif

#ifdef DEBUG
  if (proc_config[AZ_N_procs]==1)
    write_vec("rhs.dat", N_local, b);
#endif

  /* xsolve is a little longer vector needed to account for external 
     entries.  Make it and copy x (initial guess) into it. 
  */

  if (has_global_indices)
    {
      N_external = 0;
    }
  else
    {
      N_external = data_org[AZ_N_external];
    }

  xsolve  = (double *) calloc(N_local + N_external, 
			   sizeof(double)) ;

  for (i=0; i<N_local; i++) xsolve[i] = x[i];

  /* Reorder rhs and xsolve to match matrix ordering from AZ_transform */
  if (!has_global_indices)
    {
      AZ_reorder_vec(b, data_org, update_index, rpntr) ;
      AZ_reorder_vec(xsolve, data_org, update_index, rpntr) ;
    }

#ifdef VBRMATRIX
  AZ_check_vbr(N_update, data_org[AZ_N_ext_blk], AZ_LOCAL, 
	       bindx, bpntr, cpntr, rpntr, proc_config);
#else
  AZ_check_msr(bindx, N_update, N_external, AZ_LOCAL, proc_config);
#endif

  printf("Processor %d of %d N_local = %d N_external = %d NNZ = %d\n",
	 proc_config[AZ_node],proc_config[AZ_N_procs],N_local,N_external,
	 n_nonzeros);

  /* solve the system of equations using b  as the right hand side */

  Prec = AZ_precond_create(Amat,AZ_precondition, NULL);

  AZ_iterate(xsolve, b, options, params, status, proc_config,
	     Amat, Prec, NULL);
  /*AZ_ifpack_iterate(xsolve, b, options, params, status, proc_config,
    Amat);*/

  if (proc_config[AZ_node]==0)
    {
      printf("True residual norm = %22.16g\n",status[AZ_r]);
      printf("True scaled res    = %22.16g\n",status[AZ_scaled_r]);
      printf("Computed res norm  = %22.16g\n",status[AZ_rec_r]);
    }

#ifdef TEST_SINGULAR

   xnull  = (double *) calloc(N_local + N_external, sizeof(double)) ;
   Axnull  = (double *) calloc(N_local + N_external, sizeof(double)) ;
   for (i=0; i<N_local; i++) xnull[i] = xexact[i];
   if (!has_global_indices)  AZ_reorder_vec(xnull, data_org, update_index, rpntr);
   for (i=0; i<N_local; i++) xnull[i] -= xsolve[i]; /* fill with nullerence */
   Amat->matvec(xnull, Axnull, Amat, proc_config);

   norm_Axnull = AZ_gvector_norm(N_local, 2, Axnull, proc_config);

   if (proc_config[AZ_node]==0) printf("Norm of A(xexact-xsolve) = %12.4g\n",norm_Axnull);
   free((void *) xnull);
   free((void *) Axnull);
#endif


  /* Get solution back into original ordering */
   if (!has_global_indices) {
     AZ_invorder_vec(xsolve, data_org, update_index, rpntr, x);
     free((void *) xsolve);
   }
  else {
    free((void *) x);
    x = xsolve;
  }

#ifdef DEBUG
  if (proc_config[AZ_N_procs]==1)
      write_vec("solution.dat", N_local, x);
#endif
  if (xexact != NULL)
    {
      double sum = 0.0;
      double largest = 0.0;
      for (i=0; i<N_local; i++) sum += fabs(x[i]-xexact[i]);
 printf("Processor %d:  Difference between exact and computed solution = %12.4g\n",
	     proc_config[AZ_node],sum);
      for (i=0; i<N_local; i++) largest = AZ_MAX(largest,fabs(xexact[i]));
 printf("Processor %d:  Difference divided by max abs value of exact   = %12.4g\n",
	     proc_config[AZ_node],sum/largest);
    }

				       

  free((void *) val);
  free((void *) bindx);
#ifdef VBRMATRIX
  free((void *) rpntr);
  free((void *) bpntr);
  free((void *) indx);
#endif
  free((void *) b);
  free((void *) x);
  if (xexact!=NULL) free((void *) xexact);

  AZ_free((void *) update);
  AZ_free((void *) update_index);
  AZ_free((void *) external); 
  AZ_free((void *) extern_index);
  AZ_free((void *) data_org);
  if (cpntr!=NULL) AZ_free((void *) cpntr);
  AZ_precond_destroy(&Prec);
  AZ_matrix_destroy(&Amat);
  


#ifdef AZTEC_MPI
  MPI_Finalize() ;
#endif

/* end main
*/
return 0 ;
}