Exemplo n.º 1
0
int main(int argc, char *argv[])
{
  char  global[]="global";
  char  local[]="local";

  int    proc_config[AZ_PROC_SIZE];/* Processor information.                */
  int    options[AZ_OPTIONS_SIZE]; /* Array used to select solver options.  */
  double params[AZ_PARAMS_SIZE];   /* User selected solver paramters.       */
  int    *data_org;
                                   /* Array to specify data layout          */
  double status[AZ_STATUS_SIZE];   /* Information returned from AZ_solve(). */
  int    *update;                  /* vector elements updated on this node. */
  int    *external;
                                   /* vector elements needed by this node.  */
  int    *update_index;
                                   /* ordering of update[] and external[]   */
  int    *extern_index;
                                   /* locally on this processor.            */
  int    *indx;   /* MSR format of real and imag parts */
  int    *bindx;
  int    *bpntr;
  int    *rpntr;
  int    *cpntr;
  AZ_MATRIX *Amat;
  AZ_PRECOND *Prec;
  double *val;
  double *x, *b, *xexact, *xsolve;
  int    n_nonzeros, n_blk_nonzeros;
  int    N_update;           /* # of block unknowns updated on this node    */
  int    N_local;
                                 /* Number scalar equations on this node */
  int    N_global, N_blk_global; /* Total number of equations */
  int    N_external, N_blk_eqns;

  double *val_msr;
  int *bindx_msr;
  
  double norm, d ;

  int matrix_type;

  int has_global_indices, option;
  int i, j, m, mp ;
  int ione = 1;

#ifdef TEST_SINGULAR
  double * xnull; /* will contain difference of given exact solution and computed solution*/
  double * Axnull; /* Product of A time xnull */
  double norm_Axnull;
#endif

#ifdef AZTEC_MPI
  double MPI_Wtime(void) ;
#endif
  double time ;
#ifdef AZTEC_MPI
  MPI_Init(&argc,&argv);
#endif

  /* get number of processors and the name of this processor */
 
#ifdef AZTEC_MPI
  AZ_set_proc_config(proc_config,MPI_COMM_WORLD);
#else
  AZ_set_proc_config(proc_config,0);
#endif

  printf("proc %d of %d is alive\n",
	 proc_config[AZ_node],proc_config[AZ_N_procs]) ;

#ifdef AZTEC_MPI
  MPI_Barrier(MPI_COMM_WORLD) ;
#endif

#ifdef VBRMATRIX
  if(argc != 3) 
    perror("error: enter name of data and partition file on command line") ; 
#else
  if(argc != 2) perror("error: enter name of data file on command line") ; 
#endif
  /* Set exact solution to NULL */
  xexact = NULL;

  /* Read matrix file and distribute among processors.  
     Returns with this processor's set of rows */ 

#ifdef VBRMATRIX
  read_hb(argv[1], proc_config, &N_global, &n_nonzeros, 
	  &val_msr,  &bindx_msr, &x, &b, &xexact);
  
  create_vbr(argv[2], proc_config, &N_global, &N_blk_global,
	     &n_nonzeros, &n_blk_nonzeros, &N_update, &update,
	     bindx_msr, val_msr, &val, &indx, 
	     &rpntr, &cpntr, &bpntr, &bindx);

  if(proc_config[AZ_node] == 0) 
    {
      free ((void *) val_msr);
      free ((void *) bindx_msr);
      free ((void *) cpntr);
    }
    matrix_type = AZ_VBR_MATRIX;

#ifdef AZTEC_MPI
  MPI_Barrier(MPI_COMM_WORLD) ;
#endif

  distrib_vbr_matrix( proc_config, N_global, N_blk_global, 
		      &n_nonzeros, &n_blk_nonzeros,
		      &N_update, &update, 
		      &val, &indx, &rpntr, &cpntr, &bpntr, &bindx, 
		      &x, &b, &xexact);

#else
    read_hb(argv[1], proc_config, &N_global, &n_nonzeros,
             &val,  &bindx, &x, &b, &xexact);

#ifdef AZTEC_MPI
  MPI_Barrier(MPI_COMM_WORLD) ;
#endif

  distrib_msr_matrix(proc_config, N_global, &n_nonzeros, &N_update,
		  &update, &val, &bindx, &x, &b, &xexact);

#ifdef DEBUG
  for (i = 0; i<N_update; i++)
    if (val[i] == 0.0 ) printf("Zero diagonal at row %d\n",i);
#endif
    matrix_type = AZ_MSR_MATRIX;
#endif
  /* convert matrix to a local distributed matrix */
    cpntr = NULL;
  AZ_transform(proc_config, &external, bindx, val, update,
	       &update_index, &extern_index, &data_org, 
	       N_update, indx, bpntr, rpntr, &cpntr,
               matrix_type);

  printf("Processor %d: Completed AZ_transform\n",proc_config[AZ_node]) ;
      has_global_indices = 0;
      option = AZ_LOCAL;

#ifdef VBRMATRIX
  N_local = rpntr[N_update];
#else
  N_local = N_update;
#endif

  Amat = AZ_matrix_create(N_local);

#ifdef VBRMATRIX
  AZ_set_VBR(Amat, rpntr, cpntr, bpntr, indx, bindx, val, data_org,
          N_update, update, option);
#else
  AZ_set_MSR(Amat, bindx, val, data_org, N_update, update, option);
#endif


  printf("proc %d Completed AZ_create_matrix\n",proc_config[AZ_node]) ;

#ifdef AZTEC_MPI
  MPI_Barrier(MPI_COMM_WORLD) ;
#endif

  /* initialize AZTEC options */
 
  AZ_defaults(options, params);
  options[AZ_solver]  = AZ_gmres;
  options[AZ_precond] = AZ_sym_GS; 
  options[AZ_poly_ord] = 1;
  options[AZ_graph_fill] = 1;
  params[AZ_rthresh] = 0.0E-7;
  params[AZ_athresh] = 0.0E-7;
  options[AZ_overlap] = 1;
 /*
  params[AZ_ilut_fill] = 2.0;
  params[AZ_drop] = 0.01;
  options[AZ_overlap] = 0;
  options[AZ_reorder] = 0;
  params[AZ_rthresh] = 1.0E-1;
  params[AZ_athresh] = 1.0E-1;
  options[AZ_precond] = AZ_dom_decomp ;
  options[AZ_subdomain_solve] = AZ_bilu_ifp;
  options[AZ_reorder] = 0;
  options[AZ_graph_fill] = 0;
  params[AZ_rthresh] = 1.0E-7;
  params[AZ_athresh] = 1.0E-7;
 options[AZ_poly_ord] = 1;
 options[AZ_precond] = AZ_Jacobi;
  params[AZ_omega] = 1.0;
  options[AZ_precond] = AZ_none ;

  options[AZ_poly_ord] = 1;
  options[AZ_precond] = AZ_Jacobi ;
  options[AZ_scaling] = AZ_sym_row_sum ;
  options[AZ_scaling] = AZ_sym_diag;


  options[AZ_conv] = AZ_noscaled;
  options[AZ_scaling] = AZ_Jacobi ;

  options[AZ_precond] = AZ_dom_decomp ;
  options[AZ_subdomain_solve] = AZ_icc ;
  options[AZ_subdomain_solve] = AZ_ilut ;
  params[AZ_omega] = 1.2;
  params[AZ_ilut_fill] = 2.0;
  params[AZ_drop] = 0.01;
  options[AZ_reorder] = 0;
  options[AZ_overlap] = 0;
  options[AZ_type_overlap] = AZ_symmetric;

  options[AZ_precond] = AZ_dom_decomp ;
  options[AZ_subdomain_solve] = AZ_bilu ;
  options[AZ_graph_fill] = 0;
  options[AZ_overlap] = 0;

  options[AZ_precond] = AZ_dom_decomp ;
  options[AZ_subdomain_solve] = AZ_bilu_ifp ;
  options[AZ_graph_fill] = 0;
  options[AZ_overlap] = 0;
  params[AZ_rthresh] = 1.0E-3;
  params[AZ_athresh] = 1.0E-3;

 options[AZ_poly_ord] = 1;
 options[AZ_precond] = AZ_Jacobi ; */


  options[AZ_kspace] = 600 ;

  options[AZ_max_iter] = 600 ;
  params[AZ_tol] = 1.0e-14;

#ifdef BGMRES
  options[AZ_gmres_blocksize] = 3;
  options[AZ_gmres_num_rhs] = 1;
#endif

#ifdef DEBUG
  if (proc_config[AZ_N_procs]==1)
    write_vec("rhs.dat", N_local, b);
#endif

  /* xsolve is a little longer vector needed to account for external 
     entries.  Make it and copy x (initial guess) into it. 
  */

  if (has_global_indices)
    {
      N_external = 0;
    }
  else
    {
      N_external = data_org[AZ_N_external];
    }

  xsolve  = (double *) calloc(N_local + N_external, 
			   sizeof(double)) ;

  for (i=0; i<N_local; i++) xsolve[i] = x[i];

  /* Reorder rhs and xsolve to match matrix ordering from AZ_transform */
  if (!has_global_indices)
    {
      AZ_reorder_vec(b, data_org, update_index, rpntr) ;
      AZ_reorder_vec(xsolve, data_org, update_index, rpntr) ;
    }

#ifdef VBRMATRIX
  AZ_check_vbr(N_update, data_org[AZ_N_ext_blk], AZ_LOCAL, 
	       bindx, bpntr, cpntr, rpntr, proc_config);
#else
  AZ_check_msr(bindx, N_update, N_external, AZ_LOCAL, proc_config);
#endif

  printf("Processor %d of %d N_local = %d N_external = %d NNZ = %d\n",
	 proc_config[AZ_node],proc_config[AZ_N_procs],N_local,N_external,
	 n_nonzeros);

  /* solve the system of equations using b  as the right hand side */

  Prec = AZ_precond_create(Amat,AZ_precondition, NULL);

  AZ_iterate(xsolve, b, options, params, status, proc_config,
	     Amat, Prec, NULL);
  /*AZ_ifpack_iterate(xsolve, b, options, params, status, proc_config,
    Amat);*/

  if (proc_config[AZ_node]==0)
    {
      printf("True residual norm = %22.16g\n",status[AZ_r]);
      printf("True scaled res    = %22.16g\n",status[AZ_scaled_r]);
      printf("Computed res norm  = %22.16g\n",status[AZ_rec_r]);
    }

#ifdef TEST_SINGULAR

   xnull  = (double *) calloc(N_local + N_external, sizeof(double)) ;
   Axnull  = (double *) calloc(N_local + N_external, sizeof(double)) ;
   for (i=0; i<N_local; i++) xnull[i] = xexact[i];
   if (!has_global_indices)  AZ_reorder_vec(xnull, data_org, update_index, rpntr);
   for (i=0; i<N_local; i++) xnull[i] -= xsolve[i]; /* fill with nullerence */
   Amat->matvec(xnull, Axnull, Amat, proc_config);

   norm_Axnull = AZ_gvector_norm(N_local, 2, Axnull, proc_config);

   if (proc_config[AZ_node]==0) printf("Norm of A(xexact-xsolve) = %12.4g\n",norm_Axnull);
   free((void *) xnull);
   free((void *) Axnull);
#endif


  /* Get solution back into original ordering */
   if (!has_global_indices) {
     AZ_invorder_vec(xsolve, data_org, update_index, rpntr, x);
     free((void *) xsolve);
   }
  else {
    free((void *) x);
    x = xsolve;
  }

#ifdef DEBUG
  if (proc_config[AZ_N_procs]==1)
      write_vec("solution.dat", N_local, x);
#endif
  if (xexact != NULL)
    {
      double sum = 0.0;
      double largest = 0.0;
      for (i=0; i<N_local; i++) sum += fabs(x[i]-xexact[i]);
 printf("Processor %d:  Difference between exact and computed solution = %12.4g\n",
	     proc_config[AZ_node],sum);
      for (i=0; i<N_local; i++) largest = AZ_MAX(largest,fabs(xexact[i]));
 printf("Processor %d:  Difference divided by max abs value of exact   = %12.4g\n",
	     proc_config[AZ_node],sum/largest);
    }

				       

  free((void *) val);
  free((void *) bindx);
#ifdef VBRMATRIX
  free((void *) rpntr);
  free((void *) bpntr);
  free((void *) indx);
#endif
  free((void *) b);
  free((void *) x);
  if (xexact!=NULL) free((void *) xexact);

  AZ_free((void *) update);
  AZ_free((void *) update_index);
  AZ_free((void *) external); 
  AZ_free((void *) extern_index);
  AZ_free((void *) data_org);
  if (cpntr!=NULL) AZ_free((void *) cpntr);
  AZ_precond_destroy(&Prec);
  AZ_matrix_destroy(&Amat);
  


#ifdef AZTEC_MPI
  MPI_Finalize() ;
#endif

/* end main
*/
return 0 ;
}
Exemplo n.º 2
0
int main(int argc, char *argv[])
{

  /* See Aztec User's Guide for the variables that follow:         */
  int    proc_config[AZ_PROC_SIZE];/* Processor information.                */
  int    N_update;                 /* # of unknowns updated on this node    */
  int    *update;                  /* vector elements updated on this node  */

  int    *data_orgA;               /* Array to specify data layout          */
  int    *externalA;               /* vector elements needed by this node.  */
  int    *update_indexA;           /* ordering of update[] and external[]   */
  int    *extern_indexA;           /* locally on this processor.            */
  int    *bindxA;                  /* Sparse matrix to be solved is stored  */
  double *valA;                    /* in these MSR arrays.                  */
  AZ_MATRIX *mat_curl_edge;        /* curl operator matrix                  */

  int    *data_orgB;                /* Array to specify data layout          */
  int    *externalB;                /* vector elements needed by this node.  */
  int    *update_indexB;            /* ordering of update[] and external[]   */
  int    *extern_indexB;            /* locally on this processor.            */
  int    *bindxB;                   /* Sparse matrix to be solved is stored  */
  double *valB;                     /* in these MSR arrays.                  */
  AZ_MATRIX *mat_curl_face;         /* curl operator matrix                  */

  int *bc_indx;
  int n_bc;

  double *efield;
  double *bfield;
  double *epsilon;
  double *tmp_vec;
  double *tmp_vec2;

  int    i, nrow, x, y, z;
  int k, t;
  long startTime, endTime;
  int myrank;
  int vec_len;


  /* get number of processors and the name of this processor */
#ifdef AZ_MPI
  MPI_Init(&argc,&argv);
  AZ_set_proc_config(proc_config, MPI_COMM_WORLD);
  MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
#else
  myrank = 0;
  AZ_set_proc_config(proc_config, AZ_NOT_MPI);
#endif

  nrow = ncomp * nx * ny * nz;  /* overll number of matrix rows  */

  // Define partitioning:  matrix rows (ascending order) owned by this node
  // Here it is done automatically, but it can also be specified by hand
  AZ_read_update(&N_update, &update, proc_config, nrow, 1, AZ_linear);


  // In the following we set up the matrix for the edge centered curl operator
  // All the steps are described in detail in the AZTEC manual.
  // first: allocate space for the first matrix.
  bindxA = (int    *) malloc((N_update*MAX_NZ_ROW+1)*sizeof(int));
  valA   = (double *) malloc((N_update*MAX_NZ_ROW+1)*sizeof(double));
  if (valA == NULL) perror("Error: Not enough space to create matrix");
  // Initialize the index for the first off diagonal element
  bindxA[0] = N_update+1;

  // Create the matrix row by row. Each processor creates only rows appearing
  // in update[] (using global col. numbers).
  for (i = 0; i < N_update; i++)
    create_curl_matrix_row_edge(update[i], i, valA, bindxA);

  // convert matrix to a local distributed matrix
  AZ_transform(proc_config, &externalA, bindxA, valA, update, &update_indexA,
               &extern_indexA, &data_orgA, N_update, NULL, NULL, NULL, NULL,
               AZ_MSR_MATRIX);

  // convert the matrix arrays into a matrix structure, used in the
  // matrix vector multiplication
  mat_curl_edge = AZ_matrix_create(data_orgA[AZ_N_internal] + data_orgA[AZ_N_border]);
  AZ_set_MSR(mat_curl_edge, bindxA, valA, data_orgA, 0, NULL, AZ_LOCAL);

  // at this point the edge centered curl matrix is completed.

  // In the following we set up the matrix for the face centered curl operator
  // All the steps are described in detail in the AZTEC manual.
  // first: allocate space for the first matrix.
  bindxB = (int    *) malloc((N_update*MAX_NZ_ROW+1)*sizeof(int));
  valB   = (double *) malloc((N_update*MAX_NZ_ROW+1)*sizeof(double));
  if (valB == NULL) perror("Error: Not enough space to create matrix");

  // Initialize the index for the first off diagonal element
  bindxB[0] = N_update+1;

  // Create the matrix row by row. Each processor creates only rows appearing
  // in update[] (using global col. numbers).
  for (i = 0; i < N_update; i++)
      create_curl_matrix_row_face(update[i], i, valB, bindxB);

  // convert matrix to a local distributed matrix
  AZ_transform(proc_config, &externalB, bindxB, valB, update, &update_indexB,
			                   &extern_indexB, &data_orgB,
					   N_update, NULL, NULL, NULL, NULL,
					                  AZ_MSR_MATRIX);
  // convert the matrix arrays into a matrix structure, used in the
  // matrix vector multiplication
  mat_curl_face = AZ_matrix_create(data_orgB[AZ_N_internal] + data_orgB[AZ_N_border]);
  AZ_set_MSR(mat_curl_face, bindxB, valB, data_orgB, 0, NULL, AZ_LOCAL);

  // at this point the face centered curl matrix is completed.


  //  allocate memory for the fields and a temporary vector
  vec_len = N_update + data_orgA[AZ_N_external];
  efield = (double *) malloc(vec_len*sizeof(double));
  bfield = (double *) malloc(vec_len*sizeof(double));
  epsilon = (double *) malloc(vec_len*sizeof(double));
  tmp_vec = (double *) malloc(vec_len*sizeof(double));
  tmp_vec2 = (double *) malloc(vec_len*sizeof(double));

  // setup the boundary condition. We will get an arry that tells us
  // which positions need to be updated and where the results needs
  // to be stored in the E field.
  setup_bc(update, update_indexB, N_update, &bc_indx, &n_bc);

  // initialize the field vectors
  for(k = 0; k < vec_len; k++){
     efield[k] = 0.;
     bfield[k] = 0.;
     epsilon[k] = 1.;
     tmp_vec[k] = 0.;
  }

  // initialize the dielectric structure. Ugly hard-coded stuff,
  // needs to be cleaned out...
  for(y=45; y<55; y++){
   for(x = y; x<100; x++)
      epsilon[compZ + pos_to_row(x, y, 0)] = 0.95;
  }	
  // reorder the dielectric vector in order to align with the B field
  AZ_reorder_vec(epsilon, data_orgA, update_indexA, NULL);


  printf("Begin iteration \n");

  // just some timing ...
  startTime = currentTimeMillis();

  // *******************
  // begin of the time stepping loop
  // *******************

  for( t = 0; t < nsteps; t++){

    // first we do the e field update

    // convert the B field to the H field

    for(k = 0 ; k < vec_len; k++)
      bfield[k] *= epsilon[k];

    // setup the initial condition
    for( k = 0; k < n_bc; k++){
      x = bc_indx[4*k];
      y = bc_indx[4*k+1];
      z = bc_indx[4*k+2];
      efield[bc_indx[4*k+3]] =
           sin((double) y * 5. * 3.14159 / (double) ny) *
           sin(omega * dt * (double) (t + 1));
    }

    //  E field update:
    //  tmp_vec = Curl_Op * bfield
    //  efield = efield +  c^2 * dt * tmp_vec
    AZ_MSR_matvec_mult( bfield, tmp_vec, mat_curl_edge, proc_config);

    // reorder the result in tmp_vec so that it aligns with the
    // decomposition of the E field
    AZ_invorder_vec(tmp_vec, data_orgA, update_indexA, NULL, tmp_vec2);
    AZ_reorder_vec(tmp_vec2, data_orgB, update_indexB, NULL);

    // update the efield
    for(k = 0 ; k < N_update; k++)
      efield[k] = efield[k] + c2 * tmp_vec2[k] * dt;

    // bfield update :
    // tmp_vec = DualCurl_Op * efield
    // bfield = bfield - tmp_vec * dt
    AZ_MSR_matvec_mult( efield, tmp_vec, mat_curl_face, proc_config);

    // reorder the result so that it fits the decomposition of the bfield
    AZ_invorder_vec(tmp_vec, data_orgB, update_indexB, NULL, tmp_vec2);
    AZ_reorder_vec(tmp_vec2, data_orgA, update_indexA, NULL);

    // update the b field
    for(k = 0;  k < N_update; k++)
	  bfield[k] = bfield[k] - tmp_vec2[k] * dt;

    if(myrank == 0)
      printf("Taking step %d at time %g\n", t,
		       (double) (currentTimeMillis() - startTime) / 1000.);
  }
  // ******************
  // end of timestepping loop
  // *****************

  endTime = currentTimeMillis();
  printf("After iteration: %g\n", (double)(endTime - startTime) / 1000. );

#if 1
  system("rm efield.txt bfield.txt");

  // dump filed data: efield
  AZ_invorder_vec(efield, data_orgB, update_indexB, NULL, tmp_vec);
  write_file("efield.txt", tmp_vec, N_update);

  // dump filed data: bfield
  AZ_invorder_vec(bfield, data_orgA, update_indexA, NULL, tmp_vec);
  write_file("bfield.txt", tmp_vec, N_update);
#endif

  /* Free allocated memory */
  AZ_matrix_destroy( &mat_curl_edge);
  free((void *) update);   free((void *) update_indexA);
  free((void *) externalA); free((void *) extern_indexA);
  free((void *) bindxA);    free((void *) valA);  free((void *) data_orgA);

  AZ_matrix_destroy( &mat_curl_face);
  free((void *) externalB); free((void *) extern_indexB);
  free((void *) bindxB);    free((void *) valB);  free((void *) data_orgB);

  free((void *) efield);    free((void *) bfield);
  free((void *) tmp_vec);   free((void *) tmp_vec2);


#ifdef AZ_MPI
  MPI_Finalize();
#endif
  return(1);

}
Exemplo n.º 3
0
int test_azoo_conv_with_scaling(int conv_option, int scaling_option,
                                const Epetra_Comm& comm, bool verbose)
{
  int localN = 20;
  int numprocs = comm.NumProc();
  int globalN = numprocs*localN;
 
  Epetra_Map emap(globalN, 0, comm);
  Epetra_CrsMatrix* Acrs = create_and_fill_crs_matrix(emap);

  Epetra_Vector x_crs(emap), b_crs(emap);
  x_crs.PutScalar(1.0);

  Acrs->Multiply(false, x_crs, b_crs);
  x_crs.PutScalar(0.0);

  AztecOO azoo(Acrs, &x_crs, &b_crs);
  azoo.SetAztecOption(AZ_conv, conv_option);
  azoo.SetAztecOption(AZ_solver, AZ_cg);
  azoo.SetAztecOption(AZ_scaling, scaling_option);

  azoo.Iterate(100, 1.e-9);

  //now, do the same thing with 'old-fashioned Aztec', and compare
  //the solutions.

    int* proc_config = new int[AZ_PROC_SIZE];

#ifdef EPETRA_MPI
  AZ_set_proc_config(proc_config, MPI_COMM_WORLD);
  AZ_set_comm(proc_config, MPI_COMM_WORLD);
#else
  AZ_set_proc_config(proc_config, 0);
#endif

  int *external, *update_index, *external_index;
  int *external2, *update_index2, *external_index2;
  AZ_MATRIX* Amsr = NULL;
  AZ_MATRIX* Avbr = NULL;
  int err = create_and_transform_simple_matrix(AZ_MSR_MATRIX, localN, 4.0,
                                               proc_config, Amsr,
                                               external, update_index,
                                               external_index);

  int N_update = localN+Amsr->data_org[AZ_N_border];
  double* x_msr = new double[N_update];
  double* b_msr = new double[N_update*2];
  double* b_msr_u = b_msr+N_update;
  double* x_vbr = new double[N_update];
  double* b_vbr = new double[N_update*2];
  double* b_vbr_u = b_vbr+N_update;

  err = create_and_transform_simple_matrix(AZ_VBR_MATRIX, localN, 4.0,
                                           proc_config, Avbr,
                                           external2, update_index2,
                                           external_index2);
  for(int i=0; i<N_update; ++i) {
    x_msr[i] = 1.0;
    b_msr[i] = 0.0;
    b_msr_u[i] = 0.0;
    x_vbr[i] = 1.0;
    b_vbr[i] = 0.0;
    b_vbr_u[i] = 0.0;
  }

  Amsr->matvec(x_msr, b_msr, Amsr, proc_config);
  Avbr->matvec(x_vbr, b_vbr, Avbr, proc_config);

  for(int i=0; i<N_update; ++i) {
    x_msr[i] = 0.0;
    x_vbr[i] = 0.0;
  }

  //check that the rhs's are the same.
  double max_rhs_diff1 = 0.0;
  double max_rhs_diff2 = 0.0;
  double* bptr_crs = b_crs.Values();

  AZ_invorder_vec(b_msr, Amsr->data_org, update_index, NULL, b_msr_u);
  AZ_invorder_vec(b_vbr, Avbr->data_org, update_index2, Avbr->rpntr, b_vbr_u);
  for(int i=0; i<localN; ++i) {
    if (std::abs(bptr_crs[i] - b_msr_u[i]) > max_rhs_diff1) {
      max_rhs_diff1 = std::abs(bptr_crs[i] - b_msr_u[i]);
    }
    if (std::abs(bptr_crs[i] - b_vbr_u[i]) > max_rhs_diff2) {
      max_rhs_diff2 = std::abs(bptr_crs[i] - b_vbr_u[i]);
    }
  }

  if (max_rhs_diff1> 1.e-12) {
    cout << "AztecOO rhs not equal to Aztec msr rhs "<<max_rhs_diff1<<endl;
    return(-1);
  }

  if (max_rhs_diff2> 1.e-12) {
    cout << "AztecOO rhs not equal to Aztec vbr rhs "<<max_rhs_diff2<<endl;
    return(-1);
  }

  int* az_options = new int[AZ_OPTIONS_SIZE];
  double* params = new double[AZ_PARAMS_SIZE];
  double* status = new double[AZ_STATUS_SIZE];
  AZ_defaults(az_options, params);
  az_options[AZ_solver] = AZ_cg;
  az_options[AZ_conv] = conv_option;
  az_options[AZ_scaling] = scaling_option;

  az_options[AZ_max_iter] = 100;
  params[AZ_tol] = 1.e-9;

  AZ_iterate(x_msr, b_msr, az_options, params, status, proc_config,
             Amsr, NULL, NULL);
  AZ_iterate(x_vbr, b_vbr, az_options, params, status, proc_config,
             Avbr, NULL, NULL);

  AZ_invorder_vec(x_msr, Amsr->data_org, update_index, NULL, b_msr_u);
  AZ_invorder_vec(x_vbr, Avbr->data_org, update_index2, Avbr->rpntr, b_vbr_u);

  double max_diff1 = 0.0;
  double max_diff2 = 0.0;
  double* xptr_crs = x_crs.Values();

  for(int i=0; i<localN; ++i) {
    if (std::abs(xptr_crs[i] - b_msr_u[i]) > max_diff1) {
      max_diff1 = std::abs(xptr_crs[i] - b_msr_u[i]);
    }
    if (std::abs(xptr_crs[i] - b_vbr_u[i]) > max_diff2) {
      max_diff2 = std::abs(xptr_crs[i] - b_vbr_u[i]);
    }
  }

  if (max_diff1 > 1.e-7) {
    cout << "AztecOO failed to match Aztec msr with scaling and Anorm conv."
      << endl;
    return(-1);
  }

  if (max_diff2 > 1.e-7) {
    cout << "AztecOO failed to match Aztec vbr with scaling and Anorm conv."
      << endl;
    return(-1);
  }

  delete Acrs;
  delete [] x_msr;
  delete [] b_msr;
  delete [] x_vbr;
  delete [] b_vbr;
  destroy_matrix(Amsr);
  destroy_matrix(Avbr);
  delete [] proc_config;
  free(update_index);
  free(external);
  free(external_index);
  free(update_index2);
  free(external2);
  free(external_index2);
  delete [] az_options;
  delete [] params;
  delete [] status;

  return(0);
}