Beispiel #1
0
HYPRE_Int hypre_seqAMGSetup( hypre_ParAMGData *amg_data,
                      HYPRE_Int p_level,
                      HYPRE_Int coarse_threshold)


{

   /* Par Data Structure variables */
   hypre_ParCSRMatrix **Par_A_array = hypre_ParAMGDataAArray(amg_data);

   MPI_Comm 	      comm = hypre_ParCSRMatrixComm(Par_A_array[0]); 
   MPI_Comm 	      new_comm, seq_comm;

   hypre_ParCSRMatrix   *A_seq = NULL;
   hypre_CSRMatrix  *A_seq_diag;
   hypre_CSRMatrix  *A_seq_offd;
   hypre_ParVector   *F_seq = NULL;
   hypre_ParVector   *U_seq = NULL;

   hypre_ParCSRMatrix *A;

   HYPRE_Int               **dof_func_array;   
   HYPRE_Int                num_procs, my_id;

   HYPRE_Int                not_finished_coarsening;
   HYPRE_Int                level;

   HYPRE_Solver  coarse_solver;

   /* misc */
   dof_func_array = hypre_ParAMGDataDofFuncArray(amg_data);

   /*MPI Stuff */
   hypre_MPI_Comm_size(comm, &num_procs);   
   hypre_MPI_Comm_rank(comm,&my_id);
  
   /*initial */
   level = p_level;
   
   not_finished_coarsening = 1;
  
   /* convert A at this level to sequential */
   A = Par_A_array[level];

   {
      double *A_seq_data = NULL;
      HYPRE_Int *A_seq_i = NULL;
      HYPRE_Int *A_seq_offd_i = NULL;
      HYPRE_Int *A_seq_j = NULL;

      double *A_tmp_data = NULL;
      HYPRE_Int *A_tmp_i = NULL;
      HYPRE_Int *A_tmp_j = NULL;

      HYPRE_Int *info, *displs, *displs2;
      HYPRE_Int i, j, size, num_nonzeros, total_nnz, cnt;
  
      hypre_CSRMatrix *A_diag = hypre_ParCSRMatrixDiag(A);
      hypre_CSRMatrix *A_offd = hypre_ParCSRMatrixOffd(A);
      HYPRE_Int *col_map_offd = hypre_ParCSRMatrixColMapOffd(A);
      HYPRE_Int *A_diag_i = hypre_CSRMatrixI(A_diag);
      HYPRE_Int *A_offd_i = hypre_CSRMatrixI(A_offd);
      HYPRE_Int *A_diag_j = hypre_CSRMatrixJ(A_diag);
      HYPRE_Int *A_offd_j = hypre_CSRMatrixJ(A_offd);
      double *A_diag_data = hypre_CSRMatrixData(A_diag);
      double *A_offd_data = hypre_CSRMatrixData(A_offd);
      HYPRE_Int num_rows = hypre_CSRMatrixNumRows(A_diag);
      HYPRE_Int first_row_index = hypre_ParCSRMatrixFirstRowIndex(A);

      hypre_MPI_Group orig_group, new_group; 
      HYPRE_Int *ranks, new_num_procs, *row_starts;

      info = hypre_CTAlloc(HYPRE_Int, num_procs);

      hypre_MPI_Allgather(&num_rows, 1, HYPRE_MPI_INT, info, 1, HYPRE_MPI_INT, comm);

      ranks = hypre_CTAlloc(HYPRE_Int, num_procs);

      new_num_procs = 0;
      for (i=0; i < num_procs; i++)
         if (info[i]) 
         {
            ranks[new_num_procs] = i;
            info[new_num_procs++] = info[i];
         }

      MPI_Comm_group(comm, &orig_group);
      hypre_MPI_Group_incl(orig_group, new_num_procs, ranks, &new_group);
      MPI_Comm_create(comm, new_group, &new_comm);
      hypre_MPI_Group_free(&new_group);
      hypre_MPI_Group_free(&orig_group);

      if (num_rows)
      {
         /* alloc space in seq data structure only for participating procs*/
         HYPRE_BoomerAMGCreate(&coarse_solver);
         HYPRE_BoomerAMGSetMaxRowSum(coarse_solver,
		hypre_ParAMGDataMaxRowSum(amg_data)); 
         HYPRE_BoomerAMGSetStrongThreshold(coarse_solver,
		hypre_ParAMGDataStrongThreshold(amg_data)); 
         HYPRE_BoomerAMGSetCoarsenType(coarse_solver,
		hypre_ParAMGDataCoarsenType(amg_data)); 
         HYPRE_BoomerAMGSetInterpType(coarse_solver,
		hypre_ParAMGDataInterpType(amg_data)); 
         HYPRE_BoomerAMGSetTruncFactor(coarse_solver, 
		hypre_ParAMGDataTruncFactor(amg_data)); 
         HYPRE_BoomerAMGSetPMaxElmts(coarse_solver, 
		hypre_ParAMGDataPMaxElmts(amg_data)); 
	 if (hypre_ParAMGDataUserRelaxType(amg_data) > -1) 
            HYPRE_BoomerAMGSetRelaxType(coarse_solver, 
		hypre_ParAMGDataUserRelaxType(amg_data)); 
         HYPRE_BoomerAMGSetRelaxOrder(coarse_solver, 
		hypre_ParAMGDataRelaxOrder(amg_data)); 
         HYPRE_BoomerAMGSetRelaxWt(coarse_solver, 
		hypre_ParAMGDataUserRelaxWeight(amg_data)); 
	 if (hypre_ParAMGDataUserNumSweeps(amg_data) > -1) 
            HYPRE_BoomerAMGSetNumSweeps(coarse_solver, 
		hypre_ParAMGDataUserNumSweeps(amg_data)); 
         HYPRE_BoomerAMGSetNumFunctions(coarse_solver, 
		hypre_ParAMGDataNumFunctions(amg_data)); 
         HYPRE_BoomerAMGSetMaxIter(coarse_solver, 1); 
         HYPRE_BoomerAMGSetTol(coarse_solver, 0); 

         /* Create CSR Matrix, will be Diag part of new matrix */
         A_tmp_i = hypre_CTAlloc(HYPRE_Int, num_rows+1);

         A_tmp_i[0] = 0;
         for (i=1; i < num_rows+1; i++)
            A_tmp_i[i] = A_diag_i[i]-A_diag_i[i-1]+A_offd_i[i]-A_offd_i[i-1];

         num_nonzeros = A_offd_i[num_rows]+A_diag_i[num_rows];

         A_tmp_j = hypre_CTAlloc(HYPRE_Int, num_nonzeros);
         A_tmp_data = hypre_CTAlloc(double, num_nonzeros);

         cnt = 0;
         for (i=0; i < num_rows; i++)
         {
            for (j=A_diag_i[i]; j < A_diag_i[i+1]; j++)
	    {
	       A_tmp_j[cnt] = A_diag_j[j]+first_row_index;
	       A_tmp_data[cnt++] = A_diag_data[j];
	    }
            for (j=A_offd_i[i]; j < A_offd_i[i+1]; j++)
	    {
	       A_tmp_j[cnt] = col_map_offd[A_offd_j[j]];
	       A_tmp_data[cnt++] = A_offd_data[j];
	    }
         }

         displs = hypre_CTAlloc(HYPRE_Int, new_num_procs+1);
         displs[0] = 0;
         for (i=1; i < new_num_procs+1; i++)
            displs[i] = displs[i-1]+info[i-1];
         size = displs[new_num_procs];
  
         A_seq_i = hypre_CTAlloc(HYPRE_Int, size+1);
         A_seq_offd_i = hypre_CTAlloc(HYPRE_Int, size+1);

         hypre_MPI_Allgatherv ( &A_tmp_i[1], num_rows, HYPRE_MPI_INT, &A_seq_i[1], info, 
			displs, HYPRE_MPI_INT, new_comm );

         displs2 = hypre_CTAlloc(HYPRE_Int, new_num_procs+1);

         A_seq_i[0] = 0;
         displs2[0] = 0;
         for (j=1; j < displs[1]; j++)
            A_seq_i[j] = A_seq_i[j]+A_seq_i[j-1];
         for (i=1; i < new_num_procs; i++)
         {
            for (j=displs[i]; j < displs[i+1]; j++)
            {
               A_seq_i[j] = A_seq_i[j]+A_seq_i[j-1];
            }
         }
         A_seq_i[size] = A_seq_i[size]+A_seq_i[size-1];
         displs2[new_num_procs] = A_seq_i[size];
         for (i=1; i < new_num_procs+1; i++)
         {
            displs2[i] = A_seq_i[displs[i]];
            info[i-1] = displs2[i] - displs2[i-1];
         }

         total_nnz = displs2[new_num_procs];
         A_seq_j = hypre_CTAlloc(HYPRE_Int, total_nnz);
         A_seq_data = hypre_CTAlloc(double, total_nnz);

         hypre_MPI_Allgatherv ( A_tmp_j, num_nonzeros, HYPRE_MPI_INT,
                       A_seq_j, info, displs2,
                       HYPRE_MPI_INT, new_comm );

         hypre_MPI_Allgatherv ( A_tmp_data, num_nonzeros, hypre_MPI_DOUBLE,
                       A_seq_data, info, displs2,
                       hypre_MPI_DOUBLE, new_comm );

         hypre_TFree(displs);
         hypre_TFree(displs2);
         hypre_TFree(A_tmp_i);
         hypre_TFree(A_tmp_j);
         hypre_TFree(A_tmp_data);
   
         row_starts = hypre_CTAlloc(HYPRE_Int,2);
         row_starts[0] = 0; 
         row_starts[1] = size;
 
         /* Create 1 proc communicator */
         seq_comm = hypre_MPI_COMM_SELF;

         A_seq = hypre_ParCSRMatrixCreate(seq_comm,size,size,
					  row_starts, row_starts,
						0,total_nnz,0); 

         A_seq_diag = hypre_ParCSRMatrixDiag(A_seq);
         A_seq_offd = hypre_ParCSRMatrixOffd(A_seq);

         hypre_CSRMatrixData(A_seq_diag) = A_seq_data;
         hypre_CSRMatrixI(A_seq_diag) = A_seq_i;
         hypre_CSRMatrixJ(A_seq_diag) = A_seq_j;
         hypre_CSRMatrixI(A_seq_offd) = A_seq_offd_i;

         F_seq = hypre_ParVectorCreate(seq_comm, size, row_starts);
         U_seq = hypre_ParVectorCreate(seq_comm, size, row_starts);
         hypre_ParVectorOwnsPartitioning(F_seq) = 0;
         hypre_ParVectorOwnsPartitioning(U_seq) = 0;
         hypre_ParVectorInitialize(F_seq);
         hypre_ParVectorInitialize(U_seq);

         hypre_BoomerAMGSetup(coarse_solver,A_seq,F_seq,U_seq);

         hypre_ParAMGDataCoarseSolver(amg_data) = coarse_solver;
         hypre_ParAMGDataACoarse(amg_data) = A_seq;
         hypre_ParAMGDataFCoarse(amg_data) = F_seq;
         hypre_ParAMGDataUCoarse(amg_data) = U_seq;
         hypre_ParAMGDataNewComm(amg_data) = new_comm;
      }
      hypre_TFree(info);
      hypre_TFree(ranks);
   }
 
   return 0;
   
   
}
Beispiel #2
0
int main (int argc, char *argv[])
{
   int myid, num_procs;
   int n;
   double gamma, h;
   int vis;

   HYPRE_SStructGrid     grid;
   HYPRE_SStructGraph    graph;
   HYPRE_SStructMatrix   A;
   HYPRE_SStructVector   b;
   HYPRE_SStructVector   x;

   HYPRE_Solver          solver;

   /* Initialize MPI */
   MPI_Init(&argc, &argv);
   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);

   /* Set default parameters */
   n = 10;
   vis = 0;

   /* Parse command line */
   {
      int arg_index = 0;
      int print_usage = 0;

      while (arg_index < argc)
      {
         if ( strcmp(argv[arg_index], "-n") == 0 )
         {
            arg_index++;
            n = atoi(argv[arg_index++]);
         }
         else if ( strcmp(argv[arg_index], "-vis") == 0 )
         {
            arg_index++;
            vis = 1;
         }
         else if ( strcmp(argv[arg_index], "-help") == 0 )
         {
            print_usage = 1;
            break;
         }
         else
         {
            arg_index++;
         }
      }

      if ((print_usage) && (myid == 0))
      {
         printf("\n");
         printf("Usage: %s [<options>]\n", argv[0]);
         printf("\n");
         printf("  -n <n>              : problem size per processor (default: 10)\n");
         printf("  -vis                : save the solution for GLVis visualization\n");
         printf("\n");
      }

      if (print_usage)
      {
         MPI_Finalize();
         return (0);
      }
   }

   /* Set the rhombus angle, gamma, and the mesh size, h, depending on the
      number of processors np and the given n */
   if (num_procs < 3)
   {
      if (myid ==0) printf("Must run with at least 3 processors!\n");
      MPI_Finalize();
      exit(1);
   }
   gamma = 2*M_PI/num_procs;
   h = 1.0/n;

   /* 1. Set up the grid.  We will set up the grid so that processor X owns
         part X.  Note that each part has its own index space numbering. Later
         we relate the parts to each other. */
   {
      int ndim = 2;
      int nparts = num_procs;

      /* Create an empty 2D grid object */
      HYPRE_SStructGridCreate(MPI_COMM_WORLD, ndim, nparts, &grid);

      /* Set the extents of the grid - each processor sets its grid boxes.  Each
         part has its own relative index space numbering */
      {
         int part = myid;
         int ilower[2] = {1,1}; /* lower-left cell touching the origin */
         int iupper[2] = {n,n}; /* upper-right cell */

         HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
      }

      /* Set the variable type and number of variables on each part.  These need
         to be set in each part which is neighboring or contains boxes owned by
         the processor. */
      {
         int i;
         int nvars = 1;

         HYPRE_SStructVariable vartypes[1] = {HYPRE_SSTRUCT_VARIABLE_NODE};
         for (i = 0; i < nparts; i++)
            HYPRE_SStructGridSetVariables(grid, i, nvars, vartypes);
      }

      /* Set the ordering of the variables in the finite element problem.  This
         is done by listing the variable offset directions relative to the
         element's center.  See the Reference Manual for more details. */
      {
         int part = myid;
         int ordering[12] = { 0, -1, -1,    /*    [3]------[2] */
                              0, +1, -1,    /*    /        /   */
                              0, +1, +1,    /*   /        /    */
                              0, -1, +1 };  /* [0]------[1]    */

         HYPRE_SStructGridSetFEMOrdering(grid, part, ordering);
      }

      /* Now we need to set the spatial relation between each of the parts.
         Since we are using nodal variables, we have to use SetSharedPart to
         establish the connection at the origin. */
      {
         /* Relation to the clockwise-previous neighbor part, e.g. 0 and 1 for
            the case of 6 parts.  Note that we could have used SetNeighborPart
            here instead of SetSharedPart. */
         {
            int part = myid;
            /* the box of cells intersecting the boundary in the current part */
            int ilower[2] = {1,1}, iupper[2] = {1,n};
            /* share all data on the left side of the box */
            int offset[2] = {-1,0};

            int shared_part = (myid+1) % num_procs;
            /* the box of cells intersecting the boundary in the neighbor */
            int shared_ilower[2] = {1,1}, shared_iupper[2] = {n,1};
            /* share all data on the bottom of the box */
            int shared_offset[2] = {0,-1};

            /* x/y-direction on the current part is -y/x on the neighbor */
            int index_map[2] = {1,0};
            int index_dir[2] = {-1,1};

            HYPRE_SStructGridSetSharedPart(grid, part, ilower, iupper, offset,
                                           shared_part, shared_ilower,
                                           shared_iupper, shared_offset,
                                           index_map, index_dir);
         }

         /* Relation to the clockwise-following neighbor part, e.g. 0 and 5 for
            the case of 6 parts.  Note that we could have used SetNeighborPart
            here instead of SetSharedPart. */
         {
            int part = myid;
            /* the box of cells intersecting the boundary in the current part */
            int ilower[2] = {1,1}, iupper[2] = {n,1};
            /* share all data on the bottom of the box */
            int offset[2] = {0,-1};

            int shared_part = (myid+num_procs-1) % num_procs;
            /* the box of cells intersecting the boundary in the neighbor */
            int shared_ilower[2] = {1,1}, shared_iupper[2] = {1,n};
            /* share all data on the left side of the box */
            int shared_offset[2] = {-1,0};

            /* x/y-direction on the current part is y/-x on the neighbor */
            int index_map[2] = {1,0};
            int index_dir[2] = {1,-1};

            HYPRE_SStructGridSetSharedPart(grid, part, ilower, iupper, offset,
                                           shared_part, shared_ilower,
                                           shared_iupper, shared_offset,
                                           index_map, index_dir);
         }

         /* Relation to all other parts, e.g. 0 and 2,3,4.  This can be
            described only by SetSharedPart. */
         {
            int part = myid;
            /* the (one cell) box that touches the origin */
            int ilower[2] = {1,1}, iupper[2] = {1,1};
            /* share all data in the bottom left corner (i.e. the origin) */
            int offset[2] = {-1,-1};

            int shared_part;
            /* the box of one cell that touches the origin */
            int shared_ilower[2] = {1,1}, shared_iupper[2] = {1,1};
            /* share all data in the bottom left corner (i.e. the origin) */
            int shared_offset[2] = {-1,-1};

            /* x/y-direction on the current part is -x/-y on the neighbor, but
               in this case the arguments are not really important since we are
               only sharing a point */
            int index_map[2] = {0,1};
            int index_dir[2] = {-1,-1};

            for (shared_part = 0; shared_part < myid-1; shared_part++)
               HYPRE_SStructGridSetSharedPart(grid, part, ilower, iupper, offset,
                                              shared_part, shared_ilower,
                                              shared_iupper, shared_offset,
                                              index_map, index_dir);

            for (shared_part = myid+2; shared_part < num_procs; shared_part++)
               HYPRE_SStructGridSetSharedPart(grid, part, ilower, iupper, offset,
                                              shared_part, shared_ilower,
                                              shared_iupper, shared_offset,
                                              index_map, index_dir);
         }
      }

      /* Now the grid is ready to be used */
      HYPRE_SStructGridAssemble(grid);
   }

   /* 2. Set up the Graph - this determines the non-zero structure of the
         matrix. */
   {
      int part;

      /* Create the graph object */
      HYPRE_SStructGraphCreate(MPI_COMM_WORLD, grid, &graph);

      /* See MatrixSetObjectType below */
      HYPRE_SStructGraphSetObjectType(graph, HYPRE_PARCSR);

      /* Indicate that this problem uses finite element stiffness matrices and
         load vectors, instead of stencils. */
      for (part = 0; part < num_procs; part++)
         HYPRE_SStructGraphSetFEM(graph, part);

      /* The local stiffness matrix is full, so there is no need to call
         HYPRE_SStructGraphSetFEMSparsity to set its sparsity pattern. */

      /* Assemble the graph */
      HYPRE_SStructGraphAssemble(graph);
   }

   /* 3. Set up the SStruct Matrix and right-hand side vector */
   {
      int part = myid;

      /* Create the matrix object */
      HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, graph, &A);
      /* Use a ParCSR storage */
      HYPRE_SStructMatrixSetObjectType(A, HYPRE_PARCSR);
      /* Indicate that the matrix coefficients are ready to be set */
      HYPRE_SStructMatrixInitialize(A);

      /* Create an empty vector object */
      HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &b);
      /* Use a ParCSR storage */
      HYPRE_SStructVectorSetObjectType(b, HYPRE_PARCSR);
      /* Indicate that the vector coefficients are ready to be set */
      HYPRE_SStructVectorInitialize(b);

      /* Set the matrix and vector entries by finite element assembly */
      {
         /* local stifness matrix and load vector */
         double S[4][4], F[4];

         int i, j, k;
         int index[2];

         /* set the values in the interior cells */
         {
            ComputeFEMRhombus(S, F, gamma, h);

            for (i = 1; i <= n; i++)
               for (j = 1; j <= n; j++)
               {
                  index[0] = i;
                  index[1] = j;
                  HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
                  HYPRE_SStructVectorAddFEMValues(b, part, index, F);
               }
         }

         /* cells having nodes 1,2 on the domain boundary */
         {
            ComputeFEMRhombus(S, F, gamma, h);

            /* eliminate nodes 1,2 from S and F */
            for (k = 0; k < 4; k++)
            {
               S[1][k] = S[k][1] = 0.0;
               S[2][k] = S[k][2] = 0.0;
            }
            S[1][1] = 1.0;
            S[2][2] = 1.0;
            F[1] = 0.0;
            F[2] = 0.0;

            for (i = n; i <= n; i++)
               for (j = 1; j <= n; j++)
               {
                  index[0] = i;
                  index[1] = j;
                  HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
                  HYPRE_SStructVectorAddFEMValues(b, part, index, F);
               }
         }

         /* cells having nodes 2,3 on the domain boundary */
         {
            ComputeFEMRhombus(S, F, gamma, h);

            /* eliminate nodes 2,3 from S and F */
            for (k = 0; k < 4; k++)
            {
               S[2][k] = S[k][2] = 0.0;
               S[3][k] = S[k][3] = 0.0;
            }
            S[2][2] = 1.0;
            S[3][3] = 1.0;
            F[2] = 0.0;
            F[3] = 0.0;

            for (i = 1; i <= n; i++)
               for (j = n; j <= n; j++)
               {
                  index[0] = i;
                  index[1] = j;
                  HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
                  HYPRE_SStructVectorAddFEMValues(b, part, index, F);
               }

         }

         /* cells having nodes 1,2,3 on the domain boundary */
         {
            ComputeFEMRhombus(S, F, gamma, h);

            /* eliminate nodes 2,3 from S and F */
            for (k = 0; k < 4; k++)
            {
               S[1][k] = S[k][1] = 0.0;
               S[2][k] = S[k][2] = 0.0;
               S[3][k] = S[k][3] = 0.0;
            }
            S[1][1] = 1.0;
            S[2][2] = 1.0;
            S[3][3] = 1.0;
            F[1] = 0.0;
            F[2] = 0.0;
            F[3] = 0.0;

            for (i = n; i <= n; i++)
               for (j = n; j <= n; j++)
               {
                  index[0] = i;
                  index[1] = j;
                  HYPRE_SStructMatrixAddFEMValues(A, part, index, &S[0][0]);
                  HYPRE_SStructVectorAddFEMValues(b, part, index, F);
               }
         }
      }
   }

   /* Collective calls finalizing the matrix and vector assembly */
   HYPRE_SStructMatrixAssemble(A);
   HYPRE_SStructVectorAssemble(b);

   /* 4. Set up SStruct Vector for the solution vector x */
   {
      int part = myid;
      int var = 0;
      int nvalues = (n+1)*(n+1);
      double *values;

      /* Since the SetBoxValues() calls below set the values of the nodes in
         the upper-right corners of the cells, the nodal box should start
         from (0,0) instead of (1,1). */
      int ilower[2] = {0,0};
      int iupper[2] = {n,n};

      values = calloc(nvalues, sizeof(double));

      /* Create an empty vector object */
      HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &x);
      /* Set the object type to ParCSR */
      HYPRE_SStructVectorSetObjectType(x, HYPRE_PARCSR);
      /* Indicate that the vector coefficients are ready to be set */
      HYPRE_SStructVectorInitialize(x);
      /* Set the values for the initial guess */
      HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, values);

      free(values);

      /* Finalize the vector assembly */
      HYPRE_SStructVectorAssemble(x);
   }

   /* 5. Set up and call the solver (Solver options can be found in the
         Reference Manual.) */
   {
      double final_res_norm;
      int its;

      HYPRE_ParCSRMatrix    par_A;
      HYPRE_ParVector       par_b;
      HYPRE_ParVector       par_x;

      /* Extract the ParCSR objects needed in the solver */
      HYPRE_SStructMatrixGetObject(A, (void **) &par_A);
      HYPRE_SStructVectorGetObject(b, (void **) &par_b);
      HYPRE_SStructVectorGetObject(x, (void **) &par_x);

      /* Here we construct a BoomerAMG solver.  See the other SStruct examples
         as well as the Reference manual for additional solver choices. */
      HYPRE_BoomerAMGCreate(&solver);
      HYPRE_BoomerAMGSetCoarsenType(solver, 6);
      HYPRE_BoomerAMGSetStrongThreshold(solver, 0.25);
      HYPRE_BoomerAMGSetTol(solver, 1e-6);
      HYPRE_BoomerAMGSetPrintLevel(solver, 2);
      HYPRE_BoomerAMGSetMaxIter(solver, 50);

      /* call the setup */
      HYPRE_BoomerAMGSetup(solver, par_A, par_b, par_x);

      /* call the solve */
      HYPRE_BoomerAMGSolve(solver, par_A, par_b, par_x);

      /* get some info */
      HYPRE_BoomerAMGGetNumIterations(solver, &its);
      HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver,
                                                  &final_res_norm);
      /* clean up */
      HYPRE_BoomerAMGDestroy(solver);

      /* Gather the solution vector */
      HYPRE_SStructVectorGather(x);

      /* Save the solution for GLVis visualization, see vis/glvis-ex13.sh */
      if (vis)
      {
         FILE *file;
         char filename[255];

         int i, part = myid, var = 0;
         int nvalues = (n+1)*(n+1);
         double *values = calloc(nvalues, sizeof(double));
         int ilower[2] = {0,0};
         int iupper[2] = {n,n};

         /* get all local data (including a local copy of the shared values) */
         HYPRE_SStructVectorGetBoxValues(x, part, ilower, iupper,
                                         var, values);

         sprintf(filename, "%s.%06d", "vis/ex14.sol", myid);
         if ((file = fopen(filename, "w")) == NULL)
         {
            printf("Error: can't open output file %s\n", filename);
            MPI_Finalize();
            exit(1);
         }

         /* finite element space header */
         fprintf(file, "FiniteElementSpace\n");
         fprintf(file, "FiniteElementCollection: H1_2D_P1\n");
         fprintf(file, "VDim: 1\n");
         fprintf(file, "Ordering: 0\n\n");

         /* save solution */
         for (i = 0; i < nvalues; i++)
            fprintf(file, "%.14e\n", values[i]);

         fflush(file);
         fclose(file);
         free(values);

         /* save local finite element mesh */
         GLVis_PrintLocalRhombusMesh("vis/ex14.mesh", n, myid, gamma);

         /* additional visualization data */
         if (myid == 0)
         {
            sprintf(filename, "%s", "vis/ex14.data");
            file = fopen(filename, "w");
            fprintf(file, "np %d\n", num_procs);
            fflush(file);
            fclose(file);
         }
      }

      if (myid == 0)
      {
         printf("\n");
         printf("Iterations = %d\n", its);
         printf("Final Relative Residual Norm = %g\n", final_res_norm);
         printf("\n");
      }
   }

   /* Free memory */
   HYPRE_SStructGridDestroy(grid);
   HYPRE_SStructGraphDestroy(graph);
   HYPRE_SStructMatrixDestroy(A);
   HYPRE_SStructVectorDestroy(b);
   HYPRE_SStructVectorDestroy(x);

   /* Finalize MPI */
   MPI_Finalize();

   return 0;
}
Beispiel #3
0
HYPRE_Int hypre_BlockTridiagSetup(void *data, hypre_ParCSRMatrix *A,
                            hypre_ParVector *b, hypre_ParVector *x) 
{
   HYPRE_Int                i, j, *index_set1, print_level, nsweeps, relax_type;
   HYPRE_Int                nrows, nrows1, nrows2, start1, start2, *index_set2;
   HYPRE_Int                count, ierr;
   double             threshold;
   hypre_ParCSRMatrix **submatrices;
   HYPRE_Solver       precon1;
   HYPRE_Solver       precon2;
   HYPRE_IJVector     ij_u1, ij_u2, ij_f1, ij_f2;
   hypre_ParVector    *vector;
   MPI_Comm           comm;
   hypre_BlockTridiagData *b_data = (hypre_BlockTridiagData *) data;

   HYPRE_ParCSRMatrixGetComm((HYPRE_ParCSRMatrix) A, &comm);
   index_set1 = b_data->index_set1;
   nrows1 = index_set1[0];
   nrows  = hypre_ParCSRMatrixNumRows(A);
   nrows2 = nrows - nrows1;
   b_data->index_set2 = hypre_CTAlloc(HYPRE_Int, nrows2+1);
   index_set2 = b_data->index_set2;
   index_set2[0] = nrows2;
   count = 1;
   for (i = 0; i < index_set1[1]; i++) index_set2[count++] = i;
   for (i = 1; i < nrows1; i++) 
      for (j = index_set1[i]+1; j < index_set1[i+1]; j++) 
         index_set2[count++] = j;
   for (i = index_set1[nrows1]+1; i < nrows; i++) index_set2[count++] = i;

   submatrices = hypre_CTAlloc(hypre_ParCSRMatrix *, 4);
   hypre_ParCSRMatrixExtractSubmatrices(A, index_set1, &submatrices);

   nrows1 = hypre_ParCSRMatrixNumRows(submatrices[0]);
   nrows2 = hypre_ParCSRMatrixNumRows(submatrices[3]); 
   start1 = hypre_ParCSRMatrixFirstRowIndex(submatrices[0]);
   start2 = hypre_ParCSRMatrixFirstRowIndex(submatrices[3]);
   HYPRE_IJVectorCreate(comm, start1, start1+nrows1-1, &ij_u1);
   HYPRE_IJVectorSetObjectType(ij_u1, HYPRE_PARCSR);
   ierr  = HYPRE_IJVectorInitialize(ij_u1);
   ierr += HYPRE_IJVectorAssemble(ij_u1);
   hypre_assert(!ierr);
   HYPRE_IJVectorCreate(comm, start1, start1+nrows1-1, &ij_f1);
   HYPRE_IJVectorSetObjectType(ij_f1, HYPRE_PARCSR);
   ierr  = HYPRE_IJVectorInitialize(ij_f1);
   ierr += HYPRE_IJVectorAssemble(ij_f1);
   hypre_assert(!ierr);
   HYPRE_IJVectorCreate(comm, start2, start2+nrows2-1, &ij_u2);
   HYPRE_IJVectorSetObjectType(ij_u2, HYPRE_PARCSR);
   ierr  = HYPRE_IJVectorInitialize(ij_u2);
   ierr += HYPRE_IJVectorAssemble(ij_u2);
   hypre_assert(!ierr);
   HYPRE_IJVectorCreate(comm, start2, start2+nrows1-1, &ij_f2);
   HYPRE_IJVectorSetObjectType(ij_f2, HYPRE_PARCSR);
   ierr  = HYPRE_IJVectorInitialize(ij_f2);
   ierr += HYPRE_IJVectorAssemble(ij_f2);
   hypre_assert(!ierr);
   HYPRE_IJVectorGetObject(ij_f1, (void **) &vector);
   b_data->F1 = vector;
   HYPRE_IJVectorGetObject(ij_u1, (void **) &vector);
   b_data->U1 = vector;
   HYPRE_IJVectorGetObject(ij_f2, (void **) &vector);
   b_data->F2 = vector;
   HYPRE_IJVectorGetObject(ij_u2, (void **) &vector);
   b_data->U2 = vector;

   print_level = b_data->print_level;
   threshold   = b_data->threshold;
   nsweeps     = b_data->num_sweeps;
   relax_type  = b_data->relax_type;
   threshold = b_data->threshold;
   HYPRE_BoomerAMGCreate(&precon1);
   HYPRE_BoomerAMGSetMaxIter(precon1, 1);
   HYPRE_BoomerAMGSetCycleType(precon1, 1);
   HYPRE_BoomerAMGSetPrintLevel(precon1, print_level);
   HYPRE_BoomerAMGSetMaxLevels(precon1, 25);
   HYPRE_BoomerAMGSetMeasureType(precon1, 0);
   HYPRE_BoomerAMGSetCoarsenType(precon1, 0);
   HYPRE_BoomerAMGSetStrongThreshold(precon1, threshold);
   HYPRE_BoomerAMGSetNumFunctions(precon1, 1);
   HYPRE_BoomerAMGSetNumSweeps(precon1, nsweeps);
   HYPRE_BoomerAMGSetRelaxType(precon1, relax_type);
   hypre_BoomerAMGSetup(precon1, submatrices[0], b_data->U1, b_data->F1);

   HYPRE_BoomerAMGCreate(&precon2);
   HYPRE_BoomerAMGSetMaxIter(precon2, 1);
   HYPRE_BoomerAMGSetCycleType(precon2, 1);
   HYPRE_BoomerAMGSetPrintLevel(precon2, print_level);
   HYPRE_BoomerAMGSetMaxLevels(precon2, 25);
   HYPRE_BoomerAMGSetMeasureType(precon2, 0);
   HYPRE_BoomerAMGSetCoarsenType(precon2, 0);
   HYPRE_BoomerAMGSetMeasureType(precon2, 1);
   HYPRE_BoomerAMGSetStrongThreshold(precon2, threshold);
   HYPRE_BoomerAMGSetNumFunctions(precon2, 1);
   HYPRE_BoomerAMGSetNumSweeps(precon2, nsweeps);
   HYPRE_BoomerAMGSetRelaxType(precon2, relax_type);
   hypre_BoomerAMGSetup(precon2, submatrices[3], NULL, NULL);

   b_data->precon1 = precon1;
   b_data->precon2 = precon2;

   b_data->A11 = submatrices[0];
   hypre_ParCSRMatrixDestroy(submatrices[1]);
   b_data->A21 = submatrices[2];
   b_data->A22 = submatrices[3];

   hypre_TFree(submatrices);
   return (0);
}
Beispiel #4
0
 inline void numfact(unsigned int ncol, int* I, int* loc2glob, int* J, K* C) {
     static_assert(std::is_same<double, K>::value, "Hypre only supports double-precision floating-point real numbers");
     static_assert(S == 'G', "Hypre only supports nonsymmetric matrices");
     HYPRE_IJMatrixCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], loc2glob[0], loc2glob[1], &_A);
     HYPRE_IJMatrixSetObjectType(_A, HYPRE_PARCSR);
     HYPRE_IJMatrixSetRowSizes(_A, I + 1);
     _local = ncol;
     int* rows = new int[3 * _local]();
     int* diag_sizes = rows + _local;
     int* offdiag_sizes = diag_sizes + _local;
     rows[0] = I[0];
     for(unsigned int i = 0; i < _local; ++i) {
         std::for_each(J + rows[0], J + rows[0] + I[i + 1], [&](int& j) { (j < loc2glob[0] || loc2glob[1] < j) ? ++offdiag_sizes[i] : ++diag_sizes[i]; });
         rows[0] += I[i + 1];
     }
     HYPRE_IJMatrixSetDiagOffdSizes(_A, diag_sizes, offdiag_sizes);
     HYPRE_IJMatrixSetMaxOffProcElmts(_A, 0);
     HYPRE_IJMatrixInitialize(_A);
     std::iota(rows, rows + _local, loc2glob[0]);
     HYPRE_IJMatrixSetValues(_A, _local, I + 1, rows, J, C);
     HYPRE_IJMatrixAssemble(_A);
     HYPRE_IJVectorCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], &_b);
     HYPRE_IJVectorSetObjectType(_b, HYPRE_PARCSR);
     HYPRE_IJVectorInitialize(_b);
     HYPRE_IJVectorCreate(DMatrix::_communicator, loc2glob[0], loc2glob[1], &_x);
     HYPRE_IJVectorSetObjectType(_x, HYPRE_PARCSR);
     HYPRE_IJVectorInitialize(_x);
     delete [] rows;
     delete [] I;
     delete [] loc2glob;
     HYPRE_BoomerAMGCreate(_strategy == 1 ? &_solver : &_precond);
     HYPRE_BoomerAMGSetCoarsenType(_strategy == 1 ? _solver : _precond, 6); /* Falgout coarsening */
     HYPRE_BoomerAMGSetRelaxType(_strategy == 1 ? _solver : _precond, 6);   /* G-S/Jacobi hybrid relaxation */
     HYPRE_BoomerAMGSetNumSweeps(_strategy == 1 ? _solver : _precond, 1);   /* sweeps on each level */
     HYPRE_BoomerAMGSetMaxLevels(_strategy == 1 ? _solver : _precond, 10);  /* maximum number of levels */
     HYPRE_ParCSRMatrix parcsr_A;
     HYPRE_IJMatrixGetObject(_A, reinterpret_cast<void**>(&parcsr_A));
     HYPRE_ParVector par_b;
     HYPRE_IJVectorGetObject(_b, reinterpret_cast<void**>(&par_b));
     HYPRE_ParVector par_x;
     HYPRE_IJVectorGetObject(_x, reinterpret_cast<void**>(&par_x));
     if(_strategy == 1) {
         HYPRE_BoomerAMGSetTol(_solver, 1.0e-8);
         HYPRE_BoomerAMGSetMaxIter(_solver, 1000);
         HYPRE_BoomerAMGSetPrintLevel(_solver, 1);
         HYPRE_BoomerAMGSetup(_solver, parcsr_A, nullptr, nullptr);
     }
     else {
         HYPRE_BoomerAMGSetTol(_precond, 0.0);
         HYPRE_BoomerAMGSetMaxIter(_precond, 1);
         HYPRE_BoomerAMGSetPrintLevel(_precond, 1);
         if(_strategy == 2) {
             HYPRE_ParCSRPCGCreate(DMatrix::_communicator, &_solver);
             HYPRE_PCGSetMaxIter(_solver, 500);
             HYPRE_PCGSetTol(_solver, 1.0e-8);
             HYPRE_PCGSetTwoNorm(_solver, 1);
             HYPRE_PCGSetPrintLevel(_solver, 1);
             HYPRE_PCGSetLogging(_solver, 1);
             HYPRE_PCGSetPrecond(_solver, reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSolve), reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSetup), _precond);
             HYPRE_ParCSRPCGSetup(_solver, parcsr_A, par_b, par_x);
         }
         else {
             HYPRE_ParCSRFlexGMRESCreate(DMatrix::_communicator, &_solver);
             HYPRE_FlexGMRESSetKDim(_solver, 50);
             HYPRE_FlexGMRESSetMaxIter(_solver, 500);
             HYPRE_FlexGMRESSetTol(_solver, 1.0e-8);
             HYPRE_FlexGMRESSetPrintLevel(_solver, 1);
             HYPRE_FlexGMRESSetLogging(_solver, 1);
             HYPRE_FlexGMRESSetPrecond(_solver, reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSolve), reinterpret_cast<HYPRE_PtrToSolverFcn>(HYPRE_BoomerAMGSetup), _precond);
             HYPRE_ParCSRFlexGMRESSetup(_solver, parcsr_A, par_b, par_x);
         }
     }
 }
Beispiel #5
0
int main (int argc, char *argv[])
{
   HYPRE_Int i;
   int myid, num_procs;
   int N, n;

   HYPRE_Int ilower, iupper;
   HYPRE_Int local_size, extra;

   int solver_id;
   int print_solution, print_system;

   double h, h2;

   HYPRE_IJMatrix A;
   HYPRE_ParCSRMatrix parcsr_A;
   HYPRE_IJVector b;
   HYPRE_ParVector par_b;
   HYPRE_IJVector x;
   HYPRE_ParVector par_x;

   HYPRE_Solver solver, precond;

   /* Initialize MPI */
   MPI_Init(&argc, &argv);
   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);

   /* Default problem parameters */
   n = 33;
   solver_id = 0;
   print_solution  = 0;
   print_system = 0;


   /* Parse command line */
   {
      int arg_index = 0;
      int print_usage = 0;

      while (arg_index < argc)
      {
         if ( strcmp(argv[arg_index], "-n") == 0 )
         {
            arg_index++;
            n = atoi(argv[arg_index++]);
         }
         else if ( strcmp(argv[arg_index], "-solver") == 0 )
         {
            arg_index++;
            solver_id = atoi(argv[arg_index++]);
         }
         else if ( strcmp(argv[arg_index], "-print_solution") == 0 )
         {
            arg_index++;
            print_solution = 1;
         }
         else if ( strcmp(argv[arg_index], "-print_system") == 0 )
         {
            arg_index++;
            print_system = 1;
         }


         else if ( strcmp(argv[arg_index], "-help") == 0 )
         {
            print_usage = 1;
            break;
         }
         else
         {
            arg_index++;
         }
      }

      if ((print_usage) && (myid == 0))
      {
         printf("\n");
         printf("Usage: %s [<options>]\n", argv[0]);
         printf("\n");
         printf("  -n <n>              : problem size in each direction (default: 33)\n");
         printf("  -solver <ID>        : solver ID\n");
         printf("                        0  - AMG (default) \n");
         printf("                        1  - AMG-PCG\n");
         printf("                        8  - ParaSails-PCG\n");
         printf("                        50 - PCG\n");
         printf("                        61 - AMG-FlexGMRES\n");
         printf("  -print_solution     : print the solution vector\n");
         printf("  -print_system       : print the matrix and rhs\n");
         printf("\n");
      }

      if (print_usage)
      {
         MPI_Finalize();
         return (0);
      }
   }

   /* Preliminaries: want at least one processor per row */
   if (n*n < num_procs) n = sqrt(num_procs) + 1;
   N = n*n; /* global number of rows */
   h = 1.0/(n+1); /* mesh size*/
   h2 = h*h;

   /* Each processor knows only of its own rows - the range is denoted by ilower
      and upper.  Here we partition the rows. We account for the fact that
      N may not divide evenly by the number of processors. */
   local_size = N/num_procs;
   extra = N - local_size*num_procs;

   ilower = local_size*myid;
   ilower += hypre_min(myid, extra);

   iupper = local_size*(myid+1);
   iupper += hypre_min(myid+1, extra);
   iupper = iupper - 1;

   /* How many rows do I have? */
   local_size = iupper - ilower + 1;

   /* Create the matrix.
      Note that this is a square matrix, so we indicate the row partition
      size twice (since number of rows = number of cols) */
   HYPRE_IJMatrixCreate(MPI_COMM_WORLD, ilower, iupper, ilower, iupper, &A);

   /* Choose a parallel csr format storage (see the User's Manual) */
   HYPRE_IJMatrixSetObjectType(A, HYPRE_PARCSR);

   /* Initialize before setting coefficients */
   HYPRE_IJMatrixInitialize(A);

   /* Now go through my local rows and set the matrix entries.
      Each row has at most 5 entries. For example, if n=3:

      A = [M -I 0; -I M -I; 0 -I M]
      M = [4 -1 0; -1 4 -1; 0 -1 4]

      Note that here we are setting one row at a time, though
      one could set all the rows together (see the User's Manual).
   */
   {
      HYPRE_Int nnz;
      double values[5];
      HYPRE_Int cols[5];

      for (i = ilower; i <= iupper; i++)
      {
         nnz = 0;

         /* The left identity block:position i-n */
         if ((i-n)>=0)
         {
            cols[nnz] = i-n;
            values[nnz] = -1.0;
            nnz++;
         }

         /* The left -1: position i-1 */
         if (i%n)
         {
            cols[nnz] = i-1;
            values[nnz] = -1.0;
            nnz++;
         }

         /* Set the diagonal: position i */
         cols[nnz] = i;
         values[nnz] = 4.0;
         nnz++;

         /* The right -1: position i+1 */
         if ((i+1)%n)
         {
            cols[nnz] = i+1;
            values[nnz] = -1.0;
            nnz++;
         }

         /* The right identity block:position i+n */
         if ((i+n)< N)
         {
            cols[nnz] = i+n;
            values[nnz] = -1.0;
            nnz++;
         }

         /* Set the values for row i */
         HYPRE_IJMatrixSetValues(A, 1, &nnz, &i, cols, values);
      }
   }

   /* Assemble after setting the coefficients */
   HYPRE_IJMatrixAssemble(A);

   /* Note: for the testing of small problems, one may wish to read
      in a matrix in IJ format (for the format, see the output files
      from the -print_system option).
      In this case, one would use the following routine:
      HYPRE_IJMatrixRead( <filename>, MPI_COMM_WORLD,
                          HYPRE_PARCSR, &A );
      <filename>  = IJ.A.out to read in what has been printed out
      by -print_system (processor numbers are omitted).
      A call to HYPRE_IJMatrixRead is an *alternative* to the
      following sequence of HYPRE_IJMatrix calls:
      Create, SetObjectType, Initialize, SetValues, and Assemble
   */


   /* Get the parcsr matrix object to use */
   HYPRE_IJMatrixGetObject(A, (void**) &parcsr_A);


   /* Create the rhs and solution */
   HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&b);
   HYPRE_IJVectorSetObjectType(b, HYPRE_PARCSR);
   HYPRE_IJVectorInitialize(b);

   HYPRE_IJVectorCreate(MPI_COMM_WORLD, ilower, iupper,&x);
   HYPRE_IJVectorSetObjectType(x, HYPRE_PARCSR);
   HYPRE_IJVectorInitialize(x);

   /* Set the rhs values to h^2 and the solution to zero */
   {
      double *rhs_values, *x_values;
      HYPRE_Int *rows;

      rhs_values = calloc(local_size, sizeof(double));
      x_values = calloc(local_size, sizeof(double));
      rows = calloc(local_size, sizeof(HYPRE_Int));

      for (i=0; i<local_size; i++)
      {
         rhs_values[i] = h2;
         x_values[i] = 0.0;
         rows[i] = ilower + i;
      }

      HYPRE_IJVectorSetValues(b, local_size, rows, rhs_values);
      HYPRE_IJVectorSetValues(x, local_size, rows, x_values);

      free(x_values);
      free(rhs_values);
      free(rows);
   }


   HYPRE_IJVectorAssemble(b);
   /*  As with the matrix, for testing purposes, one may wish to read in a rhs:
       HYPRE_IJVectorRead( <filename>, MPI_COMM_WORLD,
                                 HYPRE_PARCSR, &b );
       as an alternative to the
       following sequence of HYPRE_IJVectors calls:
       Create, SetObjectType, Initialize, SetValues, and Assemble
   */
   HYPRE_IJVectorGetObject(b, (void **) &par_b);

   HYPRE_IJVectorAssemble(x);
   HYPRE_IJVectorGetObject(x, (void **) &par_x);


  /*  Print out the system  - files names will be IJ.out.A.XXXXX
       and IJ.out.b.XXXXX, where XXXXX = processor id */
   if (print_system)
   {
      HYPRE_IJMatrixPrint(A, "IJ.out.A");
      HYPRE_IJVectorPrint(b, "IJ.out.b");
   }


   /* Choose a solver and solve the system */

   /* AMG */
   if (solver_id == 0)
   {
      HYPRE_Int num_iterations;
      double final_res_norm;

      /* Create solver */
      HYPRE_BoomerAMGCreate(&solver);

      /* Set some parameters (See Reference Manual for more parameters) */
      HYPRE_BoomerAMGSetPrintLevel(solver, 3);  /* print solve info + parameters */
      HYPRE_BoomerAMGSetCoarsenType(solver, 6); /* Falgout coarsening */
      HYPRE_BoomerAMGSetRelaxType(solver, 3);   /* G-S/Jacobi hybrid relaxation */
      HYPRE_BoomerAMGSetNumSweeps(solver, 1);   /* Sweeeps on each level */
      HYPRE_BoomerAMGSetMaxLevels(solver, 20);  /* maximum number of levels */
      HYPRE_BoomerAMGSetTol(solver, 1e-7);      /* conv. tolerance */

      /* Now setup and solve! */
      HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
      HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);

      /* Run info - needed logging turned on */
      HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
      HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
      if (myid == 0)
      {
         printf("\n");
         printf("Iterations = %lld\n", num_iterations);
         printf("Final Relative Residual Norm = %e\n", final_res_norm);
         printf("\n");
      }

      /* Destroy solver */
      HYPRE_BoomerAMGDestroy(solver);
   }
   /* PCG */
   else if (solver_id == 50)
   {
      HYPRE_Int num_iterations;
      double final_res_norm;

      /* Create solver */
      HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);

      /* Set some parameters (See Reference Manual for more parameters) */
      HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
      HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
      HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
      HYPRE_PCGSetPrintLevel(solver, 2); /* prints out the iteration info */
      HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */

      /* Now setup and solve! */
      HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
      HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);

      /* Run info - needed logging turned on */
      HYPRE_PCGGetNumIterations(solver, &num_iterations);
      HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
      if (myid == 0)
      {
         printf("\n");
         printf("Iterations = %lld\n", num_iterations);
         printf("Final Relative Residual Norm = %e\n", final_res_norm);
         printf("\n");
      }

      /* Destroy solver */
      HYPRE_ParCSRPCGDestroy(solver);
   }
   /* PCG with AMG preconditioner */
   else if (solver_id == 1)
   {
      HYPRE_Int num_iterations;
      double final_res_norm;

      /* Create solver */
      HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);

      /* Set some parameters (See Reference Manual for more parameters) */
      HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
      HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
      HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
      HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
      HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */

      /* Now set up the AMG preconditioner and specify any parameters */
      HYPRE_BoomerAMGCreate(&precond);
      HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
      HYPRE_BoomerAMGSetCoarsenType(precond, 6);
      HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
      HYPRE_BoomerAMGSetNumSweeps(precond, 1);
      HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
      HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */

      /* Set the PCG preconditioner */
      HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
                          (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);

      /* Now setup and solve! */
      HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
      HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);

      /* Run info - needed logging turned on */
      HYPRE_PCGGetNumIterations(solver, &num_iterations);
      HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
      if (myid == 0)
      {
         printf("\n");
         printf("Iterations = %lld\n", num_iterations);
         printf("Final Relative Residual Norm = %e\n", final_res_norm);
         printf("\n");
      }

      /* Destroy solver and preconditioner */
      HYPRE_ParCSRPCGDestroy(solver);
      HYPRE_BoomerAMGDestroy(precond);
   }
   /* PCG with Parasails Preconditioner */
   else if (solver_id == 8)
   {
      HYPRE_Int num_iterations;
      double final_res_norm;

      int      sai_max_levels = 1;
      double   sai_threshold = 0.1;
      double   sai_filter = 0.05;
      int      sai_sym = 1;

      /* Create solver */
      HYPRE_ParCSRPCGCreate(MPI_COMM_WORLD, &solver);

      /* Set some parameters (See Reference Manual for more parameters) */
      HYPRE_PCGSetMaxIter(solver, 1000); /* max iterations */
      HYPRE_PCGSetTol(solver, 1e-7); /* conv. tolerance */
      HYPRE_PCGSetTwoNorm(solver, 1); /* use the two norm as the stopping criteria */
      HYPRE_PCGSetPrintLevel(solver, 2); /* print solve info */
      HYPRE_PCGSetLogging(solver, 1); /* needed to get run info later */

      /* Now set up the ParaSails preconditioner and specify any parameters */
      HYPRE_ParaSailsCreate(MPI_COMM_WORLD, &precond);

      /* Set some parameters (See Reference Manual for more parameters) */
      HYPRE_ParaSailsSetParams(precond, sai_threshold, sai_max_levels);
      HYPRE_ParaSailsSetFilter(precond, sai_filter);
      HYPRE_ParaSailsSetSym(precond, sai_sym);
      HYPRE_ParaSailsSetLogging(precond, 3);

      /* Set the PCG preconditioner */
      HYPRE_PCGSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSolve,
                          (HYPRE_PtrToSolverFcn) HYPRE_ParaSailsSetup, precond);

      /* Now setup and solve! */
      HYPRE_ParCSRPCGSetup(solver, parcsr_A, par_b, par_x);
      HYPRE_ParCSRPCGSolve(solver, parcsr_A, par_b, par_x);


      /* Run info - needed logging turned on */
      HYPRE_PCGGetNumIterations(solver, &num_iterations);
      HYPRE_PCGGetFinalRelativeResidualNorm(solver, &final_res_norm);
      if (myid == 0)
      {
         printf("\n");
         printf("Iterations = %lld\n", num_iterations);
         printf("Final Relative Residual Norm = %e\n", final_res_norm);
         printf("\n");
      }

      /* Destory solver and preconditioner */
      HYPRE_ParCSRPCGDestroy(solver);
      HYPRE_ParaSailsDestroy(precond);
   }
   /* Flexible GMRES with  AMG Preconditioner */
   else if (solver_id == 61)
   {
      HYPRE_Int num_iterations;
      double final_res_norm;
      int    restart = 30;
      int    modify = 1;


      /* Create solver */
      HYPRE_ParCSRFlexGMRESCreate(MPI_COMM_WORLD, &solver);

      /* Set some parameters (See Reference Manual for more parameters) */
      HYPRE_FlexGMRESSetKDim(solver, restart);
      HYPRE_FlexGMRESSetMaxIter(solver, 1000); /* max iterations */
      HYPRE_FlexGMRESSetTol(solver, 1e-7); /* conv. tolerance */
      HYPRE_FlexGMRESSetPrintLevel(solver, 2); /* print solve info */
      HYPRE_FlexGMRESSetLogging(solver, 1); /* needed to get run info later */


      /* Now set up the AMG preconditioner and specify any parameters */
      HYPRE_BoomerAMGCreate(&precond);
      HYPRE_BoomerAMGSetPrintLevel(precond, 1); /* print amg solution info */
      HYPRE_BoomerAMGSetCoarsenType(precond, 6);
      HYPRE_BoomerAMGSetRelaxType(precond, 6); /* Sym G.S./Jacobi hybrid */
      HYPRE_BoomerAMGSetNumSweeps(precond, 1);
      HYPRE_BoomerAMGSetTol(precond, 0.0); /* conv. tolerance zero */
      HYPRE_BoomerAMGSetMaxIter(precond, 1); /* do only one iteration! */

      /* Set the FlexGMRES preconditioner */
      HYPRE_FlexGMRESSetPrecond(solver, (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
                          (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup, precond);


      if (modify)
      /* this is an optional call  - if you don't call it, hypre_FlexGMRESModifyPCDefault
         is used - which does nothing.  Otherwise, you can define your own, similar to
         the one used here */
         HYPRE_FlexGMRESSetModifyPC( solver,
                                     (HYPRE_PtrToModifyPCFcn) hypre_FlexGMRESModifyPCAMGExample);


      /* Now setup and solve! */
      HYPRE_ParCSRFlexGMRESSetup(solver, parcsr_A, par_b, par_x);
      HYPRE_ParCSRFlexGMRESSolve(solver, parcsr_A, par_b, par_x);

      /* Run info - needed logging turned on */
      HYPRE_FlexGMRESGetNumIterations(solver, &num_iterations);
      HYPRE_FlexGMRESGetFinalRelativeResidualNorm(solver, &final_res_norm);
      if (myid == 0)
      {
         printf("\n");
         printf("Iterations = %lld\n", num_iterations);
         printf("Final Relative Residual Norm = %e\n", final_res_norm);
         printf("\n");
      }

      /* Destory solver and preconditioner */
      HYPRE_ParCSRFlexGMRESDestroy(solver);
      HYPRE_BoomerAMGDestroy(precond);

   }
   else
   {
      if (myid ==0) printf("Invalid solver id specified.\n");
   }

   /* Print the solution */
   if (print_solution)
      HYPRE_IJVectorPrint(x, "ij.out.x");

   /* Clean up */
   HYPRE_IJMatrixDestroy(A);
   HYPRE_IJVectorDestroy(b);
   HYPRE_IJVectorDestroy(x);

   /* Finalize MPI*/
   MPI_Finalize();

   return(0);
}
Beispiel #6
0
HYPRE_Int hypre_AMESetup(void *esolver)
{
   HYPRE_Int ne, *edge_bc;

   hypre_AMEData *ame_data = esolver;
   hypre_AMSData *ams_data = ame_data -> precond;

   if (ams_data -> beta_is_zero)
   {
      ame_data -> t1 = hypre_ParVectorInDomainOf(ams_data -> G);
      ame_data -> t2 = hypre_ParVectorInDomainOf(ams_data -> G);
   }
   else
   {
      ame_data -> t1 = ams_data -> r1;
      ame_data -> t2 = ams_data -> g1;
   }
   ame_data -> t3 = ams_data -> r0;

   /* Eliminate boundary conditions in G = [Gii, Gib; 0, Gbb], i.e.,
      compute [Gii, 0; 0, 0] */
   {
      HYPRE_Int i, j, k, nv;
      HYPRE_Int *offd_edge_bc;

      hypre_ParCSRMatrix *Gt;

      nv = hypre_ParCSRMatrixNumCols(ams_data -> G);
      ne = hypre_ParCSRMatrixNumRows(ams_data -> G);

      edge_bc = hypre_TAlloc(HYPRE_Int, ne);
      for (i = 0; i < ne; i++)
         edge_bc[i] = 0;

      /* Find boundary (eliminated) edges */
      {
         hypre_CSRMatrix *Ad = hypre_ParCSRMatrixDiag(ams_data -> A);
         HYPRE_Int *AdI = hypre_CSRMatrixI(Ad);
         HYPRE_Int *AdJ = hypre_CSRMatrixJ(Ad);
         HYPRE_Real *AdA = hypre_CSRMatrixData(Ad);
         hypre_CSRMatrix *Ao = hypre_ParCSRMatrixOffd(ams_data -> A);
         HYPRE_Int *AoI = hypre_CSRMatrixI(Ao);
         HYPRE_Real *AoA = hypre_CSRMatrixData(Ao);
         HYPRE_Real l1_norm;

         /* A row (edge) is boundary if its off-diag l1 norm is less than eps */
         HYPRE_Real eps = DBL_EPSILON * 1e+4;

         for (i = 0; i < ne; i++)
         {
            l1_norm = 0.0;
            for (j = AdI[i]; j < AdI[i+1]; j++)
               if (AdJ[j] != i)
                  l1_norm += fabs(AdA[j]);
            if (AoI)
               for (j = AoI[i]; j < AoI[i+1]; j++)
                  l1_norm += fabs(AoA[j]);
            if (l1_norm < eps)
               edge_bc[i] = 1;
         }
      }

      hypre_ParCSRMatrixTranspose(ams_data -> G, &Gt, 1);

      /* Use a Matvec communication to find which of the edges
         connected to local vertices are on the boundary */
      {
         hypre_ParCSRCommHandle *comm_handle;
         hypre_ParCSRCommPkg *comm_pkg;
         HYPRE_Int num_sends, *int_buf_data;
         HYPRE_Int index, start;

         offd_edge_bc = hypre_CTAlloc(HYPRE_Int, hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(Gt)));

         hypre_MatvecCommPkgCreate(Gt);
         comm_pkg = hypre_ParCSRMatrixCommPkg(Gt);

         num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
         int_buf_data = hypre_CTAlloc(HYPRE_Int,
                                      hypre_ParCSRCommPkgSendMapStart(comm_pkg,
                                                                      num_sends));
         index = 0;
         for (i = 0; i < num_sends; i++)
         {
            start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
            for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1); j++)
            {
               k = hypre_ParCSRCommPkgSendMapElmt(comm_pkg,j);
               int_buf_data[index++] = edge_bc[k];
            }
         }
         comm_handle = hypre_ParCSRCommHandleCreate(11, comm_pkg,
                                                    int_buf_data, offd_edge_bc);
         hypre_ParCSRCommHandleDestroy(comm_handle);
         hypre_TFree(int_buf_data);
      }

      /* Eliminate boundary vertex entries in G^t */
      {
         hypre_CSRMatrix *Gtd = hypre_ParCSRMatrixDiag(Gt);
         HYPRE_Int *GtdI = hypre_CSRMatrixI(Gtd);
         HYPRE_Int *GtdJ = hypre_CSRMatrixJ(Gtd);
         HYPRE_Real *GtdA = hypre_CSRMatrixData(Gtd);
         hypre_CSRMatrix *Gto = hypre_ParCSRMatrixOffd(Gt);
         HYPRE_Int *GtoI = hypre_CSRMatrixI(Gto);
         HYPRE_Int *GtoJ = hypre_CSRMatrixJ(Gto);
         HYPRE_Real *GtoA = hypre_CSRMatrixData(Gto);

         HYPRE_Int bdr;

         for (i = 0; i < nv; i++)
         {
            bdr = 0;
            /* A vertex is boundary if it belongs to a boundary edge */
            for (j = GtdI[i]; j < GtdI[i+1]; j++)
               if (edge_bc[GtdJ[j]]) { bdr = 1; break; }
            if (!bdr && GtoI)
               for (j = GtoI[i]; j < GtoI[i+1]; j++)
                  if (offd_edge_bc[GtoJ[j]]) { bdr = 1; break; }

            if (bdr)
            {
               for (j = GtdI[i]; j < GtdI[i+1]; j++)
                  /* if (!edge_bc[GtdJ[j]]) */
                  GtdA[j] = 0.0;
               if (GtoI)
                  for (j = GtoI[i]; j < GtoI[i+1]; j++)
                     /* if (!offd_edge_bc[GtoJ[j]]) */
                     GtoA[j] = 0.0;
            }
         }
      }

      hypre_ParCSRMatrixTranspose(Gt, &ame_data -> G, 1);

      hypre_ParCSRMatrixDestroy(Gt);
      hypre_TFree(offd_edge_bc);
   }

   /* Compute G^t M G */
   {
      if (!hypre_ParCSRMatrixCommPkg(ame_data -> G))
         hypre_MatvecCommPkgCreate(ame_data -> G);

      if (!hypre_ParCSRMatrixCommPkg(ame_data -> M))
         hypre_MatvecCommPkgCreate(ame_data -> M);

      hypre_BoomerAMGBuildCoarseOperator(ame_data -> G,
                                         ame_data -> M,
                                         ame_data -> G,
                                         &ame_data -> A_G);

      hypre_ParCSRMatrixFixZeroRows(ame_data -> A_G);
   }

   /* Create AMG preconditioner and PCG-AMG solver for G^tMG */
   {
      HYPRE_BoomerAMGCreate(&ame_data -> B1_G);
      HYPRE_BoomerAMGSetCoarsenType(ame_data -> B1_G, ams_data -> B_G_coarsen_type);
      HYPRE_BoomerAMGSetAggNumLevels(ame_data -> B1_G, ams_data -> B_G_agg_levels);
      HYPRE_BoomerAMGSetRelaxType(ame_data -> B1_G, ams_data -> B_G_relax_type);
      HYPRE_BoomerAMGSetNumSweeps(ame_data -> B1_G, 1);
      HYPRE_BoomerAMGSetMaxLevels(ame_data -> B1_G, 25);
      HYPRE_BoomerAMGSetTol(ame_data -> B1_G, 0.0);
      HYPRE_BoomerAMGSetMaxIter(ame_data -> B1_G, 1);
      HYPRE_BoomerAMGSetStrongThreshold(ame_data -> B1_G, ams_data -> B_G_theta);
      /* don't use exact solve on the coarsest level (matrix may be singular) */
      HYPRE_BoomerAMGSetCycleRelaxType(ame_data -> B1_G,
                                       ams_data -> B_G_relax_type,
                                       3);

      HYPRE_ParCSRPCGCreate(hypre_ParCSRMatrixComm(ame_data->A_G),
                            &ame_data -> B2_G);
      HYPRE_PCGSetPrintLevel(ame_data -> B2_G, 0);
      HYPRE_PCGSetTol(ame_data -> B2_G, 1e-12);
      HYPRE_PCGSetMaxIter(ame_data -> B2_G, 20);

      HYPRE_PCGSetPrecond(ame_data -> B2_G,
                          (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSolve,
                          (HYPRE_PtrToSolverFcn) HYPRE_BoomerAMGSetup,
                          ame_data -> B1_G);

      HYPRE_ParCSRPCGSetup(ame_data -> B2_G,
                           (HYPRE_ParCSRMatrix)ame_data->A_G,
                           (HYPRE_ParVector)ame_data->t1,
                           (HYPRE_ParVector)ame_data->t2);
   }

   /* Setup LOBPCG */
   {
      HYPRE_Int seed = 75;
      mv_InterfaceInterpreter* interpreter;
      mv_MultiVectorPtr eigenvectors;

      ame_data -> interpreter = hypre_CTAlloc(mv_InterfaceInterpreter,1);
      interpreter = (mv_InterfaceInterpreter*) ame_data -> interpreter;
      HYPRE_ParCSRSetupInterpreter(interpreter);

      ame_data -> eigenvalues = hypre_CTAlloc(HYPRE_Real, ame_data -> block_size);

      ame_data -> eigenvectors =
         mv_MultiVectorCreateFromSampleVector(interpreter,
                                              ame_data -> block_size,
                                              ame_data -> t3);
      eigenvectors = (mv_MultiVectorPtr) ame_data -> eigenvectors;

      mv_MultiVectorSetRandom (eigenvectors, seed);

      /* Make the initial vectors discretely divergence free */
      {
         HYPRE_Int i, j;
         HYPRE_Real *data;

         mv_TempMultiVector* tmp = mv_MultiVectorGetData(eigenvectors);
         HYPRE_ParVector *v = (HYPRE_ParVector*)(tmp -> vector);
         hypre_ParVector *vi;

         for (i = 0; i < ame_data -> block_size; i++)
         {
            vi = (hypre_ParVector*) v[i];
            data = hypre_VectorData(hypre_ParVectorLocalVector(vi));
            for (j = 0; j < ne; j++)
               if (edge_bc[j])
                  data[j] = 0.0;
            hypre_AMEDiscrDivFreeComponent(esolver, vi);
         }
      }
   }

   hypre_TFree(edge_bc);

   return hypre_error_flag;
}
Beispiel #7
0
int main(int argc, char *argv[]) {
  // printf("start\n");
  int i, j, pi, pj;
  int myid, num_procs;
  int Nx, Ny, nx, ny;
  //nx and ny are number of mesh cells
  int solver_id;
  int vis;
  int ilower[2], iupper[2];
  int nparts = 1;
  int part = 0;

  double hx, hy;

  HYPRE_SStructGraph graph;
  HYPRE_SStructGrid grid;
  HYPRE_SStructStencil stencil;
  HYPRE_SStructMatrix A;
  HYPRE_ParCSRMatrix parcsr_A;
  HYPRE_SStructVector b;
  HYPRE_ParVector par_b;
  HYPRE_SStructVector x;
  HYPRE_ParVector par_x;

  HYPRE_Solver solver;
  // printf("variables initialized\n");
   
  //Initialize MPI
  MPI_Init(&argc, &argv);
  MPI_Comm_rank(MPI_COMM_WORLD, &myid);
  MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
  // printf("mpi initialized\n");
   
  // Set default problem parameters
  nx = 100;
  ny = 100;
  Nx = 1;
  solver_id = 0;
  vis = 0;
  // printf("defaults set\n");
   
  // Parse command line

  int arg_index = 0;

  while (arg_index < argc) {
    if (strcmp(argv[arg_index], "-nx") == 0) {
      arg_index++;
      nx = atoi(argv[arg_index++]);
      // printf("nx set\n");
    } else if (strcmp(argv[arg_index], "-ny") == 0) {
      arg_index++;
      ny = atoi(argv[arg_index++]);
      // printf("ny set\n");
    } else if (strcmp(argv[arg_index], "-Nx") == 0) {
      arg_index++;
      Nx = atoi(argv[arg_index++]);
      // printf("Nx set\n");
    } else if (strcmp(argv[arg_index], "-solver") == 0) {
      arg_index++;
      solver_id = atoi(argv[arg_index++]);
      // printf("solver set\n");
    } else if (strcmp(argv[arg_index], "-vis") == 0) {
      arg_index++;
      vis = 1;
      // printf("print solution\n");
    } else {
      arg_index++;
      // printf("plus plus\n");
    }
  }

  // printf("command line parsed\n");
   
  // Set up processors in grid 
  Ny = num_procs / Nx;
  hx = 1.0 / (Nx * nx);
  hy = 1.0 / (Ny * ny);

  if (Ny <= Nx) {
    pj = myid / Nx;
    pi = myid - pj * Ny;
  } else {
    pi = myid / Ny;
    pj = myid - pi * Nx;
  }
  // printf("processors assigned\n");

  // Determine each processor's piece of the grid 
  ilower[0] = pi * nx;
  ilower[1] = pj * ny;

  iupper[0] = ilower[0] + nx - 1;
  iupper[1] = ilower[1] + ny - 1;

  // printf("upper and lowers assigned\n");
  HYPRE_SStructGridCreate(MPI_COMM_WORLD, 2, nparts, &grid);
  // printf("grid created\n");
  HYPRE_SStructGridSetExtents(grid, part, ilower, iupper);
  // printf("grid set\n");
  
  int nvars = 1;

  HYPRE_SStructVariable vartypes[1] = { HYPRE_SSTRUCT_VARIABLE_CELL };
  // printf("variable type set\n");
  for (i = 0; i < nparts; i++) {
    HYPRE_SStructGridSetVariables(grid, i, nvars, vartypes);
  }
  // printf("variable type set\n");
  HYPRE_SStructGridAssemble(grid);
  // printf("grid assembled\n");
  
  // Set up the stencil 
  HYPRE_SStructStencilCreate(2, 5, &stencil);
  int entry;
  int offsets[5][2] = { { 0, 0 }, { -1, 0 }, { 1, 0 }, { 0, -1 }, { 0, 1 } };
  int var = 0;

  for (entry = 0; entry < 5; entry++) {
    HYPRE_SStructStencilSetEntry(stencil, entry, offsets[entry], var);
  }

  // Create the graph 
  HYPRE_SStructGraphCreate(MPI_COMM_WORLD, grid, &graph);

  int object_type = HYPRE_PARCSR;

  HYPRE_SStructGraphSetObjectType(graph, object_type);

  HYPRE_SStructGraphSetStencil(graph, part, var, stencil);
  HYPRE_SStructGraphAssemble(graph);

  int nentries = 5;
  int nvalues = nentries * nx * ny;
  double *values;
  int stencil_indices[5];

  // Setup the matrix 
  HYPRE_SStructMatrixCreate(MPI_COMM_WORLD, graph, &A);
  HYPRE_SStructMatrixSetObjectType(A, HYPRE_PARCSR);
  HYPRE_SStructMatrixInitialize(A);

  // Set the matrix coefficients 
  values = calloc(nvalues, sizeof(double));
  for (j = 0; j < nentries; j++) {
    stencil_indices[j] = j;
  }

  // Account for differences in hx and hy 
  double hx2 = hx * hx;
  double hy2 = hy * hy;
  double hx2inv = 1 / hx2;
  double hy2inv = 1 / hy2;

  for (i = 0; i < nvalues; i += nentries) {
    values[i] = -2.0 * hx2inv - 2.0 * hy2inv;
    values[i + 1] = hx2inv;
    values[i + 2] = hx2inv;
    values[i + 3] = hy2inv;
    values[i + 4] = hy2inv;
  }

  HYPRE_SStructMatrixSetBoxValues(A, part, ilower, iupper, var, nentries,
                                  stencil_indices, values);

  // Set the dirchelet boundary condition
  {
    int bc_ilower[2];
    int bc_iupper[2];
    int nentries = 1;
    int nvaluesx = nentries * nx; /*  number of stencil entries times the length
                                      of one side of my grid box */
    int nvaluesy = nentries * ny;
    double *valuesx, *valuesy, *center_valuesx, *center_valuesy;
    int stencil_indices[1];
    int center_index[1];
    center_index[0] = 0;

    valuesx = calloc(nvaluesx, sizeof(double));
    center_valuesx = calloc(nvaluesx, sizeof(double));
    for (j = 0; j < nvaluesx; j++){
      valuesx[j] = 0.0;
      center_valuesx[j] = -hx2inv;
    }

    valuesy = calloc(nvaluesy, sizeof(double));
    center_valuesy = calloc(nvaluesy, sizeof(double));
    for (j = 0; j < nvaluesy; j++){
      valuesy[j] = 0.0;
      center_valuesy[j] = -hy2inv;
     }
 

    // Recall: pi and pj describe position in the processor grid 
    if (pj == 0) {
      // Bottom row of grid points 
      bc_ilower[0] = pi * nx;
      bc_ilower[1] = pj * ny;

      bc_iupper[0] = bc_ilower[0] + nx - 1;
      bc_iupper[1] = bc_ilower[1];

      stencil_indices[0] = 3;

      HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
                                      nentries, stencil_indices, valuesy);
      HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesy);
      // printf("Bottom row zeroed\n");
    }

    if (pj == Ny - 1) {
      // upper row of grid points 
      bc_ilower[0] = pi * nx;
      bc_ilower[1] = pj * ny + ny - 1;

      bc_iupper[0] = bc_ilower[0] + ny - 1;
      bc_iupper[1] = bc_ilower[1];

      stencil_indices[0] = 4;

      HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
                                      nentries, stencil_indices, valuesy);
      HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesy);
      // printf("Upper row zeroed\n");
    }
    if (pi == 0) {
      // Left row of grid points 
      bc_ilower[0] = pi * nx;
      bc_ilower[1] = pj * ny;

      bc_iupper[0] = bc_ilower[0];
      bc_iupper[1] = bc_ilower[1] + ny - 1;

      stencil_indices[0] = 1;

      HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
                                      nentries, stencil_indices, valuesx);
      HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesx); 
      // printf("left collumn zeroed\n");
    }

    if (pi == Nx - 1) {
      // Right row of grid points 
      bc_ilower[0] = pi * nx + nx - 1;
      bc_ilower[1] = pj * ny;

      bc_iupper[0] = bc_ilower[0];
      bc_iupper[1] = bc_ilower[1] + ny - 1;

      stencil_indices[0] = 2;

      HYPRE_SStructMatrixSetBoxValues(A, part, bc_ilower, bc_iupper, var,
                                      nentries, stencil_indices, valuesx);
      HYPRE_SStructMatrixAddToBoxValues(A, part, bc_ilower, bc_iupper, var, nentries, center_index, center_valuesx);
      // printf("right collumn zeroed\n");
    }
    free(valuesx);
    free(center_valuesx);
    free(valuesy);
    free(center_valuesy);
  }

  HYPRE_SStructMatrixAssemble(A);
  HYPRE_SStructMatrixGetObject(A, (void **)&parcsr_A);

  // Create the right hand side and solution vectors 
  HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &b);
  HYPRE_SStructVectorSetObjectType(b, HYPRE_PARCSR);
  HYPRE_SStructVectorInitialize(b);

  HYPRE_SStructVectorCreate(MPI_COMM_WORLD, grid, &x);
  HYPRE_SStructVectorSetObjectType(x, HYPRE_PARCSR);
  HYPRE_SStructVectorInitialize(x);

  // Set the right hand side, exact solution, and initial guess values 
  double *rhs_values, *x_values, *exactsolution;
  nvalues = nx * ny;

  rhs_values = calloc(nvalues, sizeof(double));
  x_values = calloc(nvalues, sizeof(double));
  exactsolution = calloc(nvalues, sizeof(double));
  // printf("RHS calculation begin\n");
    for (j = 0; j < ny; j++) {
    double y =  (ilower[1] + j + .5) * hy;
    for (i = 0; i < nx; i++) {
      double x = (ilower[0] + i + .5) * hx;
      rhs_values[i + j * nx] =
          -8. * PI2 * sin(2. * M_PI * x) * sin(2. * M_PI * y);
      exactsolution[i + j * nx] = 1.0 * sin(2. * M_PI * x) * sin(2. * M_PI * y);
      x_values[i + j * nx] = 0.0;
    }
  }
  // printf("RHS calculation ended\n");
  HYPRE_SStructVectorSetBoxValues(b, part, ilower, iupper, var, rhs_values);
  HYPRE_SStructVectorSetBoxValues(x, part, ilower, iupper, var, x_values);


  //printf("%d  %f\n", (ny*nx-1), exactsolution[ny*nx-1]);
  

  free(x_values);
  free(rhs_values);

  HYPRE_SStructVectorAssemble(b);
  // HYPRE_SStructVectorPrint("ss.initial.b", b, 0);
  HYPRE_SStructVectorGetObject(b, (void **)&par_b);
  HYPRE_SStructVectorAssemble(x);
  HYPRE_SStructVectorGetObject(x, (void **)&par_x);

  // Print out initial vectors 
  HYPRE_SStructMatrixPrint("SStructExact/ss.initial.A", A, 0);
  HYPRE_SStructVectorPrint("SStructExact/ss.initial.b", b, 0);
  // printf("About to solve\n");

  // Select a solver 
  if (solver_id == 0) {
    int num_iterations;
    double final_res_norm;

    // Create AMG solver 
    HYPRE_BoomerAMGCreate(&solver);

    // Set solver parameters 
    HYPRE_BoomerAMGSetPrintLevel(solver, 3);
    HYPRE_BoomerAMGSetCoarsenType(solver, 6);
    HYPRE_BoomerAMGSetRelaxType(solver, 3);
    HYPRE_BoomerAMGSetNumSweeps(solver, 1);
    HYPRE_BoomerAMGSetMaxLevels(solver, 20);
    HYPRE_BoomerAMGSetTol(solver, 1e-12);
    HYPRE_BoomerAMGSetMaxIter(solver, 100);
    // Set up and solve 
    HYPRE_BoomerAMGSetup(solver, parcsr_A, par_b, par_x);
    HYPRE_BoomerAMGSolve(solver, parcsr_A, par_b, par_x);
    //printf("solution obtained\n");
  
    // Run information 
    HYPRE_BoomerAMGGetNumIterations(solver, &num_iterations);
    HYPRE_BoomerAMGGetFinalRelativeResidualNorm(solver, &final_res_norm);
    if (myid == 0) {
      printf("\n");
      printf("Iterations = %d\n", num_iterations);
      printf("Final Relative Residual Norm = %e\n", final_res_norm);
      printf("\n");
    }

    // Destroy solver 
    HYPRE_BoomerAMGDestroy(solver);
    // HYPRE_ParVectorPrint(par_x, "SStructExact/ss.final.x");
    char solutionfile[255];
    sprintf(solutionfile, "%s.%06d", "ss.final.x", myid);
    HYPRE_ParVectorPrint(par_x, solutionfile);
  }

  // Save the solution for visualization and L2 norm calculation  
  if (vis) {
    FILE *file;
    FILE *solution;

    char filename[255];
    char solutionfile[255];

    sprintf(solutionfile, "%s.%06d.%d", "ss.final.x", myid, myid);

    int nvalues = nx * ny;

    double sum = 0;
    double diff, diff2, L2;

    // get the localally calculated and exact solutions)
    double *values = calloc(nvalues+1, sizeof(double));

    // Opens the local solution file, reads the solution to the array, values
    // and closes the file. 

    // Opens the solution file 
    if ((solution = fopen(solutionfile, "rt")) == NULL) {
      printf("Error: can't open output file %s\n", solutionfile);
      MPI_Finalize();
      exit(1);
    }

    // Reads the file to the array values 
    for (i = 0; i < nvalues + 1; i++) {
      fscanf(solution, "%lf", &values[i]);
    }

    // Close the file 
    fflush(solution);
    fclose(solution);

    MPI_Barrier(MPI_COMM_WORLD);

    // Calcluates the sum of the squared differences for the exact and numerical solutions on each processor   
  for (i = 1; i < nvalues + 1; i++) {
      diff = values[i] - exactsolution[i-1];
      diff2 = diff * diff;
      if (diff2 > 10){
       printf("%+6f       %+6f     %+6d\n", values[i], exactsolution[i-1],
       i-1);
      }
      sum += diff2;
      }
    //printf("Myid is: %d and mysum is: %f\n", myid, sum);
    
   //  Ensure all processors are caught up 
    MPI_Barrier(MPI_COMM_WORLD);
 
    // Passes the individual sums to processor 0 and sums them. 
    double *sendbuffer, *recvbuffer;
    sendbuffer = calloc(1,  sizeof(double));
    sendbuffer[0] = sum;
    recvbuffer = calloc(1, sizeof(double));
    int root = 0;
    int count = 1;
    MPI_Reduce(sendbuffer, recvbuffer, count, MPI_DOUBLE, MPI_SUM, root, MPI_COMM_WORLD);
    MPI_Barrier(MPI_COMM_WORLD);
 
  // Calculates the L2 norm  
  if (myid == 0){
      //printf("%f\n" , recvbuffer[myid]);
      // nvalues = (nx-2)*(ny-2);      
      double totalvalues = nvalues * num_procs * 1.0; 

      L2 = sqrt(recvbuffer[0] / totalvalues);      
      printf("The L2 norm is %e\n", L2);    
    }
   
    sprintf(filename, "%s.%06d", "vis/ss.sol", myid);

    if ((file = fopen(filename, "w")) == NULL) {
      printf("Error: can't open output file %s\n", filename);
      MPI_Finalize();
      exit(1);
    }

    // Save solution 
    for (i = 1; i < nvalues+1; i++) {
      fprintf(file, "%.14e\n", values[i]);
    }

    // Vis Cleanup
    fflush(file);
    fclose(file);

    free(values);
    free(exactsolution);
    free(sendbuffer);
    free(recvbuffer);

    // save global finite element mesh 
    if (myid == 0) {
      GLVis_PrintGlobalMesh("vis/ss.mesh", Nx, Ny, nx, ny, hx, hy);
    }
 } 

  // Clean up 

  HYPRE_SStructMatrixDestroy(A);
  HYPRE_SStructVectorDestroy(b);
  HYPRE_SStructVectorDestroy(x);

  // Finalize MPI*/
  
  MPI_Finalize();

  return (0);
}