示例#1
0
HYPRE_Int
hypre_SemiInterp( void               *interp_vdata,
                  hypre_StructMatrix *P,
                  hypre_StructVector *xc,
                  hypre_StructVector *e            )
{
   hypre_SemiInterpData   *interp_data = interp_vdata;

   HYPRE_Int               P_stored_as_transpose;
   hypre_ComputePkg       *compute_pkg;
   hypre_IndexRef          cindex;
   hypre_IndexRef          findex;
   hypre_IndexRef          stride;

   hypre_StructGrid       *fgrid;
   HYPRE_Int              *fgrid_ids;
   hypre_StructGrid       *cgrid;
   hypre_BoxArray         *cgrid_boxes;
   HYPRE_Int              *cgrid_ids;

   hypre_CommHandle       *comm_handle;
                       
   hypre_BoxArrayArray    *compute_box_aa;
   hypre_BoxArray         *compute_box_a;
   hypre_Box              *compute_box;
                       
   hypre_Box              *P_dbox;
   hypre_Box              *xc_dbox;
   hypre_Box              *e_dbox;
                       
   HYPRE_Int               Pi;
   HYPRE_Int               xci;
   HYPRE_Int               ei;
   HYPRE_Int               constant_coefficient;
                         
   HYPRE_Real             *Pp0, *Pp1;
   HYPRE_Real             *xcp;
   HYPRE_Real             *ep, *ep0, *ep1;
                       
   hypre_Index             loop_size;
   hypre_Index             start;
   hypre_Index             startc;
   hypre_Index             stridec;
                       
   hypre_StructStencil    *stencil;
   hypre_Index            *stencil_shape;

   HYPRE_Int               compute_i, fi, ci, j;

   /*-----------------------------------------------------------------------
    * Initialize some things
    *-----------------------------------------------------------------------*/

   hypre_BeginTiming(interp_data -> time_index);

   P_stored_as_transpose = (interp_data -> P_stored_as_transpose);
   compute_pkg   = (interp_data -> compute_pkg);
   cindex        = (interp_data -> cindex);
   findex        = (interp_data -> findex);
   stride        = (interp_data -> stride);

   stencil       = hypre_StructMatrixStencil(P);
   stencil_shape = hypre_StructStencilShape(stencil);
   constant_coefficient = hypre_StructMatrixConstantCoefficient(P);
   hypre_assert( constant_coefficient==0 || constant_coefficient==1 );
   /* ... constant_coefficient==2 for P shouldn't happen, see
      hypre_PFMGCreateInterpOp in pfmg_setup_interp.c */

   if (constant_coefficient) hypre_StructVectorClearBoundGhostValues(e, 0);

   hypre_SetIndex3(stridec, 1, 1, 1);

   /*-----------------------------------------------------------------------
    * Compute e at coarse points (injection)
    *-----------------------------------------------------------------------*/

   fgrid = hypre_StructVectorGrid(e);
   fgrid_ids = hypre_StructGridIDs(fgrid);
   cgrid = hypre_StructVectorGrid(xc);
   cgrid_boxes = hypre_StructGridBoxes(cgrid);
   cgrid_ids = hypre_StructGridIDs(cgrid);

   fi = 0;
   hypre_ForBoxI(ci, cgrid_boxes)
   {
      while (fgrid_ids[fi] != cgrid_ids[ci])
      {
         fi++;
      }

      compute_box = hypre_BoxArrayBox(cgrid_boxes, ci);

      hypre_CopyIndex(hypre_BoxIMin(compute_box), startc);
      hypre_StructMapCoarseToFine(startc, cindex, stride, start);

      e_dbox  = hypre_BoxArrayBox(hypre_StructVectorDataSpace(e), fi);
      xc_dbox = hypre_BoxArrayBox(hypre_StructVectorDataSpace(xc), ci);

      ep  = hypre_StructVectorBoxData(e, fi);
      xcp = hypre_StructVectorBoxData(xc, ci);

      hypre_BoxGetSize(compute_box, loop_size);

      hypre_BoxLoop2Begin(hypre_StructMatrixNDim(P), loop_size,
                          e_dbox, start, stride, ei,
                          xc_dbox, startc, stridec, xci);
#ifdef HYPRE_USING_OPENMP
#pragma omp parallel for private(HYPRE_BOX_PRIVATE,ei,xci) HYPRE_SMP_SCHEDULE
#endif
      hypre_BoxLoop2For(ei, xci)
      {
         ep[ei] = xcp[xci];
      }
      hypre_BoxLoop2End(ei, xci);
   }

   /*-----------------------------------------------------------------------
    * Compute e at fine points
    *-----------------------------------------------------------------------*/

   for (compute_i = 0; compute_i < 2; compute_i++)
   {
      switch(compute_i)
      {
         case 0:
         {
            ep = hypre_StructVectorData(e);
            hypre_InitializeIndtComputations(compute_pkg, ep, &comm_handle);
            compute_box_aa = hypre_ComputePkgIndtBoxes(compute_pkg);
         }
         break;

         case 1:
         {
            hypre_FinalizeIndtComputations(comm_handle);
            compute_box_aa = hypre_ComputePkgDeptBoxes(compute_pkg);
         }
         break;
      }

      hypre_ForBoxArrayI(fi, compute_box_aa)
      {
         compute_box_a = hypre_BoxArrayArrayBoxArray(compute_box_aa, fi);

         P_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(P), fi);
         e_dbox = hypre_BoxArrayBox(hypre_StructVectorDataSpace(e), fi);

         if (P_stored_as_transpose)
         {
            if ( constant_coefficient )
            {
               Pp0 = hypre_StructMatrixBoxData(P, fi, 1);
               Pp1 = hypre_StructMatrixBoxData(P, fi, 0) -
                  hypre_CCBoxOffsetDistance(P_dbox, stencil_shape[0]);
            }
            else
            {
               Pp0 = hypre_StructMatrixBoxData(P, fi, 1);
               Pp1 = hypre_StructMatrixBoxData(P, fi, 0) -
                  hypre_BoxOffsetDistance(P_dbox, stencil_shape[0]);
            }
         }
         else
         {
            Pp0 = hypre_StructMatrixBoxData(P, fi, 0);
            Pp1 = hypre_StructMatrixBoxData(P, fi, 1);
         }
         ep  = hypre_StructVectorBoxData(e, fi);
         ep0 = ep + hypre_BoxOffsetDistance(e_dbox, stencil_shape[0]);
         ep1 = ep + hypre_BoxOffsetDistance(e_dbox, stencil_shape[1]);

         hypre_ForBoxI(j, compute_box_a)
         {
            compute_box = hypre_BoxArrayBox(compute_box_a, j);

            hypre_CopyIndex(hypre_BoxIMin(compute_box), start);
            hypre_StructMapFineToCoarse(start, findex, stride, startc);

            hypre_BoxGetStrideSize(compute_box, stride, loop_size);

            if ( constant_coefficient )
            {
               Pi = hypre_CCBoxIndexRank( P_dbox, startc );
               hypre_BoxLoop1Begin(hypre_StructMatrixNDim(P), loop_size,
                                   e_dbox, start, stride, ei);
#ifdef HYPRE_USING_OPENMP
#pragma omp parallel for private(HYPRE_BOX_PRIVATE,ei) HYPRE_SMP_SCHEDULE
#endif
               hypre_BoxLoop1For(ei)
               {
                  ep[ei] =  (Pp0[Pi] * ep0[ei] +
                             Pp1[Pi] * ep1[ei]);
               }
               hypre_BoxLoop1End(ei);
            }
            else
            {
               hypre_BoxLoop2Begin(hypre_StructMatrixNDim(P), loop_size,
                                   P_dbox, startc, stridec, Pi,
                                   e_dbox, start, stride, ei);
#ifdef HYPRE_USING_OPENMP
#pragma omp parallel for private(HYPRE_BOX_PRIVATE,Pi,ei) HYPRE_SMP_SCHEDULE
#endif
               hypre_BoxLoop2For(Pi, ei)
               {
                  ep[ei] =  (Pp0[Pi] * ep0[ei] +
                             Pp1[Pi] * ep1[ei]);
               }
               hypre_BoxLoop2End(Pi, ei);
            }
         }
示例#2
0
HYPRE_Int
hypre_PFMGBuildCoarseOp5( hypre_StructMatrix *A,
                          hypre_StructMatrix *P,
                          hypre_StructMatrix *R,
                          HYPRE_Int           cdir,
                          hypre_Index         cindex,
                          hypre_Index         cstride,
                          hypre_StructMatrix *RAP     )
{
   hypre_Index           index;
   hypre_Index           index_temp;

   hypre_StructGrid     *fgrid;
   hypre_BoxArray       *fgrid_boxes;
   hypre_Box            *fgrid_box;
   HYPRE_Int            *fgrid_ids;
   hypre_StructGrid     *cgrid;
   hypre_BoxArray       *cgrid_boxes;
   hypre_Box            *cgrid_box;
   HYPRE_Int            *cgrid_ids;
   hypre_IndexRef        cstart, bfstart, stridef;
   hypre_Index           fstart, bcstart, stridec;
   hypre_Index           loop_size;

   HYPRE_Int             constant_coefficient;

   HYPRE_Int             fi, ci, fbi;
   HYPRE_Int             loopi, loopj, loopk;

   hypre_Box            *A_dbox;
   hypre_Box            *P_dbox;
   hypre_Box            *RAP_dbox;

   hypre_BoxArray       *bdy_boxes, *tmp_boxes;
   hypre_Box            *bdy_box, *fcbox;

   double               *pb, *pa;

   double               *a_cc, *a_cw, *a_ce, *a_cb, *a_ca;

   double               *rap_cc, *rap_cw, *rap_ce;
   double               *rap_cb, *rap_ca;
   double                west, east;
   double                center_int, center_bdy;

   HYPRE_Int             iA, iAm1, iAp1;
   HYPRE_Int             iAc;
   HYPRE_Int             iP, iPm1, iPp1;
                      
   HYPRE_Int             OffsetA; 
   HYPRE_Int             OffsetP; 
                      
   stridef = cstride;
   hypre_SetIndex(stridec, 1, 1, 1);

   fgrid = hypre_StructMatrixGrid(A);
   fgrid_boxes = hypre_StructGridBoxes(fgrid);
   fgrid_ids = hypre_StructGridIDs(fgrid);

   cgrid = hypre_StructMatrixGrid(RAP);
   cgrid_boxes = hypre_StructGridBoxes(cgrid);
   cgrid_ids = hypre_StructGridIDs(cgrid);

   constant_coefficient = hypre_StructMatrixConstantCoefficient(RAP);
   hypre_assert( hypre_StructMatrixConstantCoefficient(A) == constant_coefficient );
   if ( constant_coefficient==0 )
   {
      hypre_assert( hypre_StructMatrixConstantCoefficient(R) == 0 );
      hypre_assert( hypre_StructMatrixConstantCoefficient(P) == 0 );
   }
   else /* 1 or 2 */
   {
      hypre_assert( hypre_StructMatrixConstantCoefficient(R) == 1 );
      hypre_assert( hypre_StructMatrixConstantCoefficient(P) == 1 );
   }

   fcbox = hypre_BoxCreate();
   bdy_boxes = hypre_BoxArrayCreate(0);
   tmp_boxes = hypre_BoxArrayCreate(0);

   fi = 0;
   hypre_ForBoxI(ci, cgrid_boxes)
   {
      while (fgrid_ids[fi] != cgrid_ids[ci])
      {
         fi++;
      }

      cgrid_box = hypre_BoxArrayBox(cgrid_boxes, ci);
      fgrid_box = hypre_BoxArrayBox(fgrid_boxes, fi);

      cstart = hypre_BoxIMin(cgrid_box);
      hypre_StructMapCoarseToFine(cstart, cindex, cstride, fstart);

      A_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(A), fi);
      P_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(P), fi);
      RAP_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(RAP), ci);

      /*-----------------------------------------------------------------
       * Extract pointers for interpolation operator:
       * pb is pointer for weight for f-point below c-point 
       * pa is pointer for weight for f-point above c-point 
       *-----------------------------------------------------------------*/

      hypre_SetIndex(index_temp,0,-1,0);
      MapIndex(index_temp, cdir, index);
      pa = hypre_StructMatrixExtractPointerByIndex(P, fi, index);

      hypre_SetIndex(index_temp,0,1,0);
      MapIndex(index_temp, cdir, index);
      pb = hypre_StructMatrixExtractPointerByIndex(P, fi, index) -
         hypre_BoxOffsetDistance(P_dbox, index);
 
      /*-----------------------------------------------------------------
       * Extract pointers for 5-point fine grid operator:
       * 
       * a_cc is pointer for center coefficient
       * a_cw is pointer for west coefficient
       * a_ce is pointer for east coefficient
       * a_cb is pointer for below coefficient
       * a_ca is pointer for above coefficient
       *-----------------------------------------------------------------*/

      hypre_SetIndex(index_temp,0,0,0);
      MapIndex(index_temp, cdir, index);
      a_cc = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

      hypre_SetIndex(index_temp,-1,0,0);
      MapIndex(index_temp, cdir, index);
      a_cw = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

      hypre_SetIndex(index_temp,1,0,0);
      MapIndex(index_temp, cdir, index);
      a_ce = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

      hypre_SetIndex(index_temp,0,-1,0);
      MapIndex(index_temp, cdir, index);
      a_cb = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

      hypre_SetIndex(index_temp,0,1,0);
      MapIndex(index_temp, cdir, index);
      a_ca = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

      /*-----------------------------------------------------------------
       * Extract pointers for coarse grid operator
       * rap_cc is pointer for center coefficient (etc.)
       *-----------------------------------------------------------------*/

      hypre_SetIndex(index_temp,0,0,0);
      MapIndex(index_temp, cdir, index);
      rap_cc = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

      hypre_SetIndex(index_temp,-1,0,0);
      MapIndex(index_temp, cdir, index);
      rap_cw = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

      hypre_SetIndex(index_temp,1,0,0);
      MapIndex(index_temp, cdir, index);
      rap_ce = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

      hypre_SetIndex(index_temp,0,-1,0);
      MapIndex(index_temp, cdir, index);
      rap_cb = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

      hypre_SetIndex(index_temp,0,1,0);
      MapIndex(index_temp, cdir, index);
      rap_ca = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

      /*-----------------------------------------------------------------
       * Define offsets for fine grid stencil and interpolation
       *
       * In the BoxLoop below I assume iA and iP refer to data associated
       * with the point which we are building the stencil for. The below
       * Offsets are used in refering to data associated with other points. 
       *-----------------------------------------------------------------*/

      hypre_SetIndex(index_temp,0,1,0);
      MapIndex(index_temp, cdir, index);

      OffsetP = hypre_BoxOffsetDistance(P_dbox,index);
      OffsetA = hypre_BoxOffsetDistance(A_dbox,index);

      /*--------------------------------------------------------------
       * Loop for symmetric 5-point fine grid operator; produces a
       * symmetric 5-point coarse grid operator. 
       *--------------------------------------------------------------*/

      if ( constant_coefficient==0 )
      {
         hypre_BoxGetSize(cgrid_box, loop_size);

         hypre_BoxLoop3Begin(loop_size,
                             P_dbox, cstart, stridec, iP,
                             A_dbox, fstart, stridef, iA,
                             RAP_dbox, cstart, stridec, iAc);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,iP,iA,iAc,iAm1,iAp1,iPm1,iPp1,west,east
#include "hypre_box_smp_forloop.h"
         hypre_BoxLoop3For(loopi, loopj, loopk, iP, iA, iAc)
         {
            iAm1 = iA - OffsetA;
            iAp1 = iA + OffsetA;

            iPm1 = iP - OffsetP;
            iPp1 = iP + OffsetP;

            rap_cb[iAc] = a_cb[iA] * pa[iPm1];
            rap_ca[iAc] = a_ca[iA] * pb[iPp1];

            west  = a_cw[iA] + 0.5 * a_cw[iAm1] + 0.5 * a_cw[iAp1];
            east  = a_ce[iA] + 0.5 * a_ce[iAm1] + 0.5 * a_ce[iAp1];

            /*-----------------------------------------------------
             * Prevent non-zero entries reaching off grid
             *-----------------------------------------------------*/
            if(a_cw[iA] == 0.0) west = 0.0;
            if(a_ce[iA] == 0.0) east = 0.0;

            rap_cw[iAc] = west;
            rap_ce[iAc] = east;

            rap_cc[iAc] = a_cc[iA] + a_cw[iA] + a_ce[iA]
               + a_cb[iA] * pb[iP] + a_ca[iA] * pa[iP]
               - west - east;
         }
         hypre_BoxLoop3End(iP, iA, iAc);
      }
示例#3
0
HYPRE_Int
hypre_SparseMSGInterp( void               *interp_vdata,
                       hypre_StructMatrix *P,
                       hypre_StructVector *xc,
                       hypre_StructVector *e            )
{
   HYPRE_Int ierr = 0;

   hypre_SparseMSGInterpData   *interp_data = interp_vdata;

   hypre_ComputePkg       *compute_pkg;
   hypre_IndexRef          cindex;
   hypre_IndexRef          findex;
   hypre_IndexRef          stride;
   hypre_IndexRef          strideP;

   hypre_StructGrid       *fgrid;
   HYPRE_Int              *fgrid_ids;
   hypre_StructGrid       *cgrid;
   hypre_BoxArray         *cgrid_boxes;
   HYPRE_Int              *cgrid_ids;

   hypre_CommHandle       *comm_handle;
                       
   hypre_BoxArrayArray    *compute_box_aa;
   hypre_BoxArray         *compute_box_a;
   hypre_Box              *compute_box;
                       
   hypre_Box              *P_dbox;
   hypre_Box              *xc_dbox;
   hypre_Box              *e_dbox;
                       
   HYPRE_Int               Pi;
   HYPRE_Int               xci;
   HYPRE_Int               ei;
                         
   double                 *Pp0, *Pp1;
   double                 *xcp;
   double                 *ep, *ep0, *ep1;
                       
   hypre_Index             loop_size;
   hypre_Index             start;
   hypre_Index             startc;
   hypre_Index             startP;
   hypre_Index             stridec;
                       
   hypre_StructStencil    *stencil;
   hypre_Index            *stencil_shape;

   HYPRE_Int               compute_i, fi, ci, j;
   HYPRE_Int               loopi, loopj, loopk;

   /*-----------------------------------------------------------------------
    * Initialize some things
    *-----------------------------------------------------------------------*/

   hypre_BeginTiming(interp_data -> time_index);

   compute_pkg   = (interp_data -> compute_pkg);
   cindex        = (interp_data -> cindex);
   findex        = (interp_data -> findex);
   stride        = (interp_data -> stride);
   strideP       = (interp_data -> strideP);

   stencil       = hypre_StructMatrixStencil(P);
   stencil_shape = hypre_StructStencilShape(stencil);

   hypre_SetIndex(stridec, 1, 1, 1);

   /*-----------------------------------------------------------------------
    * Compute e at coarse points (injection)
    *-----------------------------------------------------------------------*/

   fgrid = hypre_StructVectorGrid(e);
   fgrid_ids = hypre_StructGridIDs(fgrid);
   cgrid = hypre_StructVectorGrid(xc);
   cgrid_boxes = hypre_StructGridBoxes(cgrid);
   cgrid_ids = hypre_StructGridIDs(cgrid);

   fi = 0;
   hypre_ForBoxI(ci, cgrid_boxes)
      {
         while (fgrid_ids[fi] != cgrid_ids[ci])
         {
            fi++;
         }

         compute_box = hypre_BoxArrayBox(cgrid_boxes, ci);

         hypre_CopyIndex(hypre_BoxIMin(compute_box), startc);
         hypre_StructMapCoarseToFine(startc, cindex, stride, start);

         e_dbox  = hypre_BoxArrayBox(hypre_StructVectorDataSpace(e), fi);
         xc_dbox = hypre_BoxArrayBox(hypre_StructVectorDataSpace(xc), ci);

         ep  = hypre_StructVectorBoxData(e, fi);
         xcp = hypre_StructVectorBoxData(xc, ci);

         hypre_BoxGetSize(compute_box, loop_size);

         hypre_BoxLoop2Begin(loop_size,
                             e_dbox,  start,  stride,  ei,
                             xc_dbox, startc, stridec, xci);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,ei,xci
#include "hypre_box_smp_forloop.h"
         hypre_BoxLoop2For(loopi, loopj, loopk, ei, xci)
            {
               ep[ei] = xcp[xci];
            }
         hypre_BoxLoop2End(ei, xci);
      }

   /*-----------------------------------------------------------------------
    * Compute e at fine points
    *-----------------------------------------------------------------------*/

   for (compute_i = 0; compute_i < 2; compute_i++)
   {
      switch(compute_i)
      {
         case 0:
         {
            ep = hypre_StructVectorData(e);
            hypre_InitializeIndtComputations(compute_pkg, ep, &comm_handle);
            compute_box_aa = hypre_ComputePkgIndtBoxes(compute_pkg);
         }
         break;

         case 1:
         {
            hypre_FinalizeIndtComputations(comm_handle);
            compute_box_aa = hypre_ComputePkgDeptBoxes(compute_pkg);
         }
         break;
      }

      hypre_ForBoxArrayI(fi, compute_box_aa)
         {
            compute_box_a = hypre_BoxArrayArrayBoxArray(compute_box_aa, fi);

            P_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(P), fi);
            e_dbox = hypre_BoxArrayBox(hypre_StructVectorDataSpace(e), fi);

            Pp0 = hypre_StructMatrixBoxData(P, fi, 0);
            Pp1 = hypre_StructMatrixBoxData(P, fi, 1);
            ep  = hypre_StructVectorBoxData(e, fi);
            ep0 = ep + hypre_BoxOffsetDistance(e_dbox, stencil_shape[0]);
            ep1 = ep + hypre_BoxOffsetDistance(e_dbox, stencil_shape[1]);

            hypre_ForBoxI(j, compute_box_a)
               {
                  compute_box = hypre_BoxArrayBox(compute_box_a, j);

                  hypre_CopyIndex(hypre_BoxIMin(compute_box), start);
                  hypre_StructMapFineToCoarse(start,  findex, stride,  startc);
                  hypre_StructMapCoarseToFine(startc, cindex, strideP, startP);

                  hypre_BoxGetStrideSize(compute_box, stride, loop_size);

                  hypre_BoxLoop2Begin(loop_size,
                                      P_dbox, startP, strideP, Pi,
                                      e_dbox, start,  stride,  ei);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,Pi,ei
#include "hypre_box_smp_forloop.h"
                  hypre_BoxLoop2For(loopi, loopj, loopk, Pi, ei)
                     {
                        ep[ei] =  (Pp0[Pi] * ep0[ei] +
                                   Pp1[Pi] * ep1[ei]);
                     }
                  hypre_BoxLoop2End(Pi, ei);
               }
         }
int
hypre_SMGResidual( void               *residual_vdata,
                   hypre_StructMatrix *A,
                   hypre_StructVector *x,
                   hypre_StructVector *b,
                   hypre_StructVector *r              )
{
   int ierr = 0;

   hypre_SMGResidualData  *residual_data = residual_vdata;

   hypre_IndexRef          base_stride = (residual_data -> base_stride);
   hypre_BoxArray         *base_points = (residual_data -> base_points);
   hypre_ComputePkg       *compute_pkg = (residual_data -> compute_pkg);

   hypre_CommHandle       *comm_handle;
                       
   hypre_BoxArrayArray    *compute_box_aa;
   hypre_BoxArray         *compute_box_a;
   hypre_Box              *compute_box;
                       
   hypre_Box              *A_data_box;
   hypre_Box              *x_data_box;
   hypre_Box              *b_data_box;
   hypre_Box              *r_data_box;
                       
   int                     Ai;
   int                     xi;
   int                     bi;
   int                     ri;
                         
   double                 *Ap;
   double                 *xp;
   double                 *bp;
   double                 *rp;
                       
   hypre_Index             loop_size;
   hypre_IndexRef          start;
                       
   hypre_StructStencil    *stencil;
   hypre_Index            *stencil_shape;
   int                     stencil_size;

   int                     compute_i, i, j, si;
   int                     loopi, loopj, loopk;

   hypre_BeginTiming(residual_data -> time_index);

   /*-----------------------------------------------------------------------
    * Compute residual r = b - Ax
    *-----------------------------------------------------------------------*/

   stencil       = hypre_StructMatrixStencil(A);
   stencil_shape = hypre_StructStencilShape(stencil);
   stencil_size  = hypre_StructStencilSize(stencil);

   for (compute_i = 0; compute_i < 2; compute_i++)
   {
      switch(compute_i)
      {
         case 0:
         {
            xp = hypre_StructVectorData(x);
            hypre_InitializeIndtComputations(compute_pkg, xp, &comm_handle);
            compute_box_aa = hypre_ComputePkgIndtBoxes(compute_pkg);

            /*----------------------------------------
             * Copy b into r
             *----------------------------------------*/

            compute_box_a = base_points;
            hypre_ForBoxI(i, compute_box_a)
               {
                  compute_box = hypre_BoxArrayBox(compute_box_a, i);
                  start = hypre_BoxIMin(compute_box);

                  b_data_box =
                     hypre_BoxArrayBox(hypre_StructVectorDataSpace(b), i);
                  r_data_box =
                     hypre_BoxArrayBox(hypre_StructVectorDataSpace(r), i);

                  bp = hypre_StructVectorBoxData(b, i);
                  rp = hypre_StructVectorBoxData(r, i);

                  hypre_BoxGetStrideSize(compute_box, base_stride, loop_size);
                  hypre_BoxLoop2Begin(loop_size,
                                      b_data_box, start, base_stride, bi,
                                      r_data_box, start, base_stride, ri);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,bi,ri
#include "hypre_box_smp_forloop.h"
                  hypre_BoxLoop2For(loopi, loopj, loopk, bi, ri)
                     {
                        rp[ri] = bp[bi];
                     }
                  hypre_BoxLoop2End(bi, ri);
               }
         }
         break;

         case 1:
         {
            hypre_FinalizeIndtComputations(comm_handle);
            compute_box_aa = hypre_ComputePkgDeptBoxes(compute_pkg);
         }
         break;
      }

      /*--------------------------------------------------------------------
       * Compute r -= A*x
       *--------------------------------------------------------------------*/

      hypre_ForBoxArrayI(i, compute_box_aa)
         {
            compute_box_a = hypre_BoxArrayArrayBoxArray(compute_box_aa, i);

            A_data_box = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(A), i);
            x_data_box = hypre_BoxArrayBox(hypre_StructVectorDataSpace(x), i);
            r_data_box = hypre_BoxArrayBox(hypre_StructVectorDataSpace(r), i);

            rp = hypre_StructVectorBoxData(r, i);

            hypre_ForBoxI(j, compute_box_a)
               {
                  compute_box = hypre_BoxArrayBox(compute_box_a, j);

                  start  = hypre_BoxIMin(compute_box);

                  for (si = 0; si < stencil_size; si++)
                  {
                     Ap = hypre_StructMatrixBoxData(A, i, si);
                     xp = hypre_StructVectorBoxData(x, i) +
                        hypre_BoxOffsetDistance(x_data_box, stencil_shape[si]);

                     hypre_BoxGetStrideSize(compute_box, base_stride,
                                            loop_size);
                     hypre_BoxLoop3Begin(loop_size,
                                         A_data_box, start, base_stride, Ai,
                                         x_data_box, start, base_stride, xi,
                                         r_data_box, start, base_stride, ri);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,Ai,xi,ri
#include "hypre_box_smp_forloop.h"
                     hypre_BoxLoop3For(loopi, loopj, loopk, Ai, xi, ri)
                        {
                           rp[ri] -= Ap[Ai] * xp[xi];
                        }
                     hypre_BoxLoop3End(Ai, xi, ri);
                  }
int
hypre_SMG2BuildRAPSym( hypre_StructMatrix *A,
                       hypre_StructMatrix *PT,
                       hypre_StructMatrix *R,
                       hypre_StructMatrix *RAP,
                       hypre_Index         cindex,
                       hypre_Index         cstride )

{

   hypre_Index           index;

   hypre_StructStencil  *fine_stencil;
   int                   fine_stencil_size;

   hypre_StructGrid     *fgrid;
   int                  *fgrid_ids;
   hypre_StructGrid     *cgrid;
   hypre_BoxArray       *cgrid_boxes;
   int                  *cgrid_ids;
   hypre_Box            *cgrid_box;
   hypre_IndexRef        cstart;
   hypre_Index           stridec;
   hypre_Index           fstart;
   hypre_IndexRef        stridef;
   hypre_Index           loop_size;

   int                   fi, ci;
   int                   loopi, loopj, loopk;

   hypre_Box            *A_dbox;
   hypre_Box            *PT_dbox;
   hypre_Box            *R_dbox;
   hypre_Box            *RAP_dbox;

   double               *pa, *pb;
   double               *ra, *rb;

   double               *a_cc, *a_cw, *a_ce, *a_cs, *a_cn;
   double               *a_csw, *a_cse, *a_cnw;

   double               *rap_cc, *rap_cw, *rap_cs;
   double               *rap_csw, *rap_cse;

   int                  iA, iAm1, iAp1;
   int                  iAc;
   int                  iP, iP1;
   int                  iR;

   int                  yOffsetA; 
   int                  xOffsetP; 
   int                  yOffsetP; 

   int                  ierr = 0;

   fine_stencil = hypre_StructMatrixStencil(A);
   fine_stencil_size = hypre_StructStencilSize(fine_stencil);

   stridef = cstride;
   hypre_SetIndex(stridec, 1, 1, 1);

   fgrid = hypre_StructMatrixGrid(A);
   fgrid_ids = hypre_StructGridIDs(fgrid);

   cgrid = hypre_StructMatrixGrid(RAP);
   cgrid_boxes = hypre_StructGridBoxes(cgrid);
   cgrid_ids = hypre_StructGridIDs(cgrid);

   fi = 0;
   hypre_ForBoxI(ci, cgrid_boxes)
      {
         while (fgrid_ids[fi] != cgrid_ids[ci])
         {
            fi++;
         }

         cgrid_box = hypre_BoxArrayBox(cgrid_boxes, ci);

         cstart = hypre_BoxIMin(cgrid_box);
         hypre_StructMapCoarseToFine(cstart, cindex, cstride, fstart);

         A_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(A), fi);
         PT_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(PT), fi);
         R_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(R), fi);
         RAP_dbox = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(RAP), ci);

         /*-----------------------------------------------------------------
          * Extract pointers for interpolation operator:
          * pa is pointer for weight for f-point above c-point 
          * pb is pointer for weight for f-point below c-point 
          *-----------------------------------------------------------------*/

         hypre_SetIndex(index,0,1,0);
         pa = hypre_StructMatrixExtractPointerByIndex(PT, fi, index);

         hypre_SetIndex(index,0,-1,0);
         pb = hypre_StructMatrixExtractPointerByIndex(PT, fi, index);
 
         /*-----------------------------------------------------------------
          * Extract pointers for restriction operator:
          * ra is pointer for weight for f-point above c-point 
          * rb is pointer for weight for f-point below c-point 
          *-----------------------------------------------------------------*/

         hypre_SetIndex(index,0,1,0);
         ra = hypre_StructMatrixExtractPointerByIndex(R, fi, index);

         hypre_SetIndex(index,0,-1,0);
         rb = hypre_StructMatrixExtractPointerByIndex(R, fi, index);
 
         /*-----------------------------------------------------------------
          * Extract pointers for 5-point fine grid operator:
          * 
          * a_cc is pointer for center coefficient
          * a_cw is pointer for west coefficient
          * a_ce is pointer for east coefficient
          * a_cs is pointer for south coefficient
          * a_cn is pointer for north coefficient
          *-----------------------------------------------------------------*/

         hypre_SetIndex(index,0,0,0);
         a_cc = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

         hypre_SetIndex(index,-1,0,0);
         a_cw = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

         hypre_SetIndex(index,1,0,0);
         a_ce = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

         hypre_SetIndex(index,0,-1,0);
         a_cs = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

         hypre_SetIndex(index,0,1,0);
         a_cn = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

         /*-----------------------------------------------------------------
          * Extract additional pointers for 9-point fine grid operator:
          *
          * a_csw is pointer for southwest coefficient
          * a_cse is pointer for southeast coefficient
          * a_cnw is pointer for northwest coefficient
          * a_cne is pointer for northeast coefficient
          *-----------------------------------------------------------------*/

         if(fine_stencil_size > 5)
         {
            hypre_SetIndex(index,-1,-1,0);
            a_csw = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

            hypre_SetIndex(index,1,-1,0);
            a_cse = hypre_StructMatrixExtractPointerByIndex(A, fi, index);

            hypre_SetIndex(index,-1,1,0);
            a_cnw = hypre_StructMatrixExtractPointerByIndex(A, fi, index);
         }

         /*-----------------------------------------------------------------
          * Extract pointers for coarse grid operator - always 9-point:
          *
          * We build only the lower triangular part (plus diagonal).
          * 
          * rap_cc is pointer for center coefficient (etc.)
          *-----------------------------------------------------------------*/

         hypre_SetIndex(index,0,0,0);
         rap_cc = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

         hypre_SetIndex(index,-1,0,0);
         rap_cw = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

         hypre_SetIndex(index,0,-1,0);
         rap_cs = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

         hypre_SetIndex(index,-1,-1,0);
         rap_csw = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

         hypre_SetIndex(index,1,-1,0);
         rap_cse = hypre_StructMatrixExtractPointerByIndex(RAP, ci, index);

         /*-----------------------------------------------------------------
          * Define offsets for fine grid stencil and interpolation
          *
          * In the BoxLoop below I assume iA and iP refer to data associated
          * with the point which we are building the stencil for. The below
          * Offsets are used in refering to data associated with other points. 
          *-----------------------------------------------------------------*/

         hypre_SetIndex(index,0,1,0);
         yOffsetA = hypre_BoxOffsetDistance(A_dbox,index); 
         yOffsetP = hypre_BoxOffsetDistance(PT_dbox,index); 
         hypre_SetIndex(index,1,0,0);
         xOffsetP = hypre_BoxOffsetDistance(PT_dbox,index); 

         /*-----------------------------------------------------------------
          * Switch statement to direct control to apropriate BoxLoop depending
          * on stencil size. Default is full 9-point.
          *-----------------------------------------------------------------*/

         switch (fine_stencil_size)
         {

            /*--------------------------------------------------------------
             * Loop for symmetric 5-point fine grid operator; produces a
             * symmetric 9-point coarse grid operator. We calculate only the
             * lower triangular stencil entries: (southwest, south, southeast,
             * west, and center).
             *--------------------------------------------------------------*/

            case 5:

            hypre_BoxGetSize(cgrid_box, loop_size);
            hypre_BoxLoop4Begin(loop_size,
                                PT_dbox,  cstart, stridec, iP,
                                R_dbox,   cstart, stridec, iR,
                                A_dbox,   fstart, stridef, iA,
                                RAP_dbox, cstart, stridec, iAc);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,iP,iR,iA,iAc,iAm1,iAp1,iP1
#include "hypre_box_smp_forloop.h"
            hypre_BoxLoop4For(loopi, loopj, loopk, iP, iR, iA, iAc)
               {
                  iAm1 = iA - yOffsetA;
                  iAp1 = iA + yOffsetA;

                  iP1 = iP - yOffsetP - xOffsetP;
                  rap_csw[iAc] = rb[iR] * a_cw[iAm1] * pa[iP1];

                  iP1 = iP - yOffsetP;
                  rap_cs[iAc] = rb[iR] * a_cc[iAm1] * pa[iP1]
                     +          rb[iR] * a_cs[iAm1]
                     +                   a_cs[iA]   * pa[iP1];

                  iP1 = iP - yOffsetP + xOffsetP;
                  rap_cse[iAc] = rb[iR] * a_ce[iAm1] * pa[iP1];

                  iP1 = iP - xOffsetP;
                  rap_cw[iAc] =          a_cw[iA]
                     +          rb[iR] * a_cw[iAm1] * pb[iP1]
                     +          ra[iR] * a_cw[iAp1] * pa[iP1];

                  rap_cc[iAc] =          a_cc[iA]
                     +          rb[iR] * a_cc[iAm1] * pb[iP]
                     +          ra[iR] * a_cc[iAp1] * pa[iP]
                     +          rb[iR] * a_cn[iAm1]
                     +          ra[iR] * a_cs[iAp1]
                     +                   a_cs[iA]   * pb[iP]
                     +                   a_cn[iA]   * pa[iP];

               }
            hypre_BoxLoop4End(iP, iR, iA, iAc);

            break;

            /*--------------------------------------------------------------
             * Loop for symmetric 9-point fine grid operator; produces a
             * symmetric 9-point coarse grid operator. We calculate only the
             * lower triangular stencil entries: (southwest, south, southeast,
             * west, and center).
             *--------------------------------------------------------------*/

            default:

            hypre_BoxGetSize(cgrid_box, loop_size);
            hypre_BoxLoop4Begin(loop_size,
                                PT_dbox,  cstart, stridec, iP,
                                R_dbox,   cstart, stridec, iR,
                                A_dbox,   fstart, stridef, iA,
                                RAP_dbox, cstart, stridec, iAc);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,iP,iR,iA,iAc,iAm1,iAp1,iP1
#include "hypre_box_smp_forloop.h"
            hypre_BoxLoop4For(loopi, loopj, loopk, iP, iR, iA, iAc)
               {
                  iAm1 = iA - yOffsetA;
                  iAp1 = iA + yOffsetA;

                  iP1 = iP - yOffsetP - xOffsetP;
                  rap_csw[iAc] = rb[iR] * a_cw[iAm1] * pa[iP1]
                     +           rb[iR] * a_csw[iAm1]
                     +                    a_csw[iA]  * pa[iP1];

                  iP1 = iP - yOffsetP;
                  rap_cs[iAc] = rb[iR] * a_cc[iAm1] * pa[iP1]
                     +          rb[iR] * a_cs[iAm1]
                     +                   a_cs[iA]   * pa[iP1];

                  iP1 = iP - yOffsetP + xOffsetP;
                  rap_cse[iAc] = rb[iR] * a_ce[iAm1] * pa[iP1]
                     +           rb[iR] * a_cse[iAm1]
                     +                    a_cse[iA]  * pa[iP1];

                  iP1 = iP - xOffsetP;
                  rap_cw[iAc] =          a_cw[iA]
                     +          rb[iR] * a_cw[iAm1] * pb[iP1]
                     +          ra[iR] * a_cw[iAp1] * pa[iP1]
                     +          rb[iR] * a_cnw[iAm1]
                     +          ra[iR] * a_csw[iAp1]
                     +                   a_csw[iA]  * pb[iP1]
                     +                   a_cnw[iA]  * pa[iP1];

                  rap_cc[iAc] =          a_cc[iA]
                     +          rb[iR] * a_cc[iAm1] * pb[iP]
                     +          ra[iR] * a_cc[iAp1] * pa[iP]
                     +          rb[iR] * a_cn[iAm1]
                     +          ra[iR] * a_cs[iAp1]
                     +                   a_cs[iA]   * pb[iP]
                     +                   a_cn[iA]   * pa[iP];

               }
            hypre_BoxLoop4End(iP, iR, iA, iAc);

            break;

         } /* end switch statement */
int
hypre_SMGResidual( void               *residual_vdata,
                   hypre_StructMatrix *A,
                   hypre_StructVector *x,
                   hypre_StructVector *b,
                   hypre_StructVector *r              )
{
   int ierr = 0;

   hypre_SMGResidualData *residual_data = (hypre_SMGResidualData *)residual_vdata;

   hypre_IndexRef          base_stride = (residual_data -> base_stride);
   hypre_BoxArray         *base_points = (residual_data -> base_points);
   hypre_ComputePkg       *compute_pkg = (residual_data -> compute_pkg);

   hypre_CommHandle       *comm_handle;
                       
   hypre_BoxArrayArray    *compute_box_aa;
   hypre_BoxArray         *compute_box_a;
   hypre_Box              *compute_box;
                       
   hypre_Box              *A_data_box;
   hypre_Box              *x_data_box;
   hypre_Box              *b_data_box;
   hypre_Box              *r_data_box;
                       
   int                     Ai;
   int                     xi;
   int                     bi;
   int                     ri;
                         
   double                 *Ap;
   double                 *xp;
   double                 *bp;
   double                 *rp;
                       
   hypre_Index             loop_size;
   hypre_IndexRef          start;
                       
   hypre_StructStencil    *stencil;
   hypre_Index            *stencil_shape;
   int                     stencil_size;

   int                     compute_i, i, j, si;
   int                     loopi, loopj, loopk;

   hypre_BeginTiming(residual_data -> time_index);

   /*-----------------------------------------------------------------------
    * Compute residual r = b - Ax
    *-----------------------------------------------------------------------*/

   stencil       = hypre_StructMatrixStencil(A);
   stencil_shape = hypre_StructStencilShape(stencil);
   stencil_size  = hypre_StructStencilSize(stencil);

   for (compute_i = 0; compute_i < 2; compute_i++)
   {
      switch(compute_i)
      {
         case 0:
         {
            xp = hypre_StructVectorData(x);
            hypre_InitializeIndtComputations(compute_pkg, xp, &comm_handle);
            compute_box_aa = hypre_ComputePkgIndtBoxes(compute_pkg);

            /*----------------------------------------
             * Copy b into r
             *----------------------------------------*/

            compute_box_a = base_points;
            hypre_ForBoxI(i, compute_box_a)
               {
                  compute_box = hypre_BoxArrayBox(compute_box_a, i);
                  start = hypre_BoxIMin(compute_box);

                  b_data_box =
                     hypre_BoxArrayBox(hypre_StructVectorDataSpace(b), i);
                  r_data_box =
                     hypre_BoxArrayBox(hypre_StructVectorDataSpace(r), i);

                  bp = hypre_StructVectorBoxData(b, i);
                  rp = hypre_StructVectorBoxData(r, i);

                  hypre_BoxGetStrideSize(compute_box, base_stride, loop_size);
                  hypre_BoxLoop2Begin(loop_size,
                                      b_data_box, start, base_stride, bi,
                                      r_data_box, start, base_stride, ri);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,bi,ri
#include "hypre_box_smp_forloop.h"
                  hypre_BoxLoop2For(loopi, loopj, loopk, bi, ri)
                     {
                        rp[ri] = bp[bi];
                     }
                  hypre_BoxLoop2End(bi, ri);
               }
         }
         break;

         case 1:
         {
            hypre_FinalizeIndtComputations(comm_handle);
            compute_box_aa = hypre_ComputePkgDeptBoxes(compute_pkg);
         }
         break;
      }

      /*--------------------------------------------------------------------
       * Compute r -= A*x
       *--------------------------------------------------------------------*/

      hypre_ForBoxArrayI(i, compute_box_aa)
         {
            compute_box_a = hypre_BoxArrayArrayBoxArray(compute_box_aa, i);

            A_data_box = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(A), i);
            x_data_box = hypre_BoxArrayBox(hypre_StructVectorDataSpace(x), i);
            r_data_box = hypre_BoxArrayBox(hypre_StructVectorDataSpace(r), i);

            rp = hypre_StructVectorBoxData(r, i);

            hypre_ForBoxI(j, compute_box_a)
               {
                  compute_box = hypre_BoxArrayBox(compute_box_a, j);

                  start  = hypre_BoxIMin(compute_box);

                  for (si = 0; si < stencil_size; si++)
                  {
                     Ap = hypre_StructMatrixBoxData(A, i, si);
                     xp = hypre_StructVectorBoxData(x, i) +
                        hypre_BoxOffsetDistance(x_data_box, stencil_shape[si]);

                     hypre_BoxGetStrideSize(compute_box, base_stride,
                                            loop_size);
                     hypre_BoxLoop3Begin(loop_size,
                                         A_data_box, start, base_stride, Ai,
                                         x_data_box, start, base_stride, xi,
                                         r_data_box, start, base_stride, ri);
#if 0                     
/*  The following portion is preprocessed to be handled by ROSE outliner */
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,Ai,xi,ri
#include "hypre_box_smp_forloop.h"
                     hypre_BoxLoop3For(loopi, loopj, loopk, Ai, xi, ri)
                        {
                           rp[ri] -= Ap[Ai] * xp[xi];
                        }
                     hypre_BoxLoop3End(Ai, xi, ri);
#else
                    for (hypre__block = 0; hypre__block < hypre__num_blocks;
                         hypre__block++)
                      {
                        loopi = 0;
                        loopj = 0;
                        loopk = 0;
                        hypre__nx = hypre__mx;
                        hypre__ny = hypre__my;
                        hypre__nz = hypre__mz;
                        if (hypre__num_blocks > 1)
                          {
                            if (hypre__dir == 0)
                              {
                                loopi =
                                  hypre__block * hypre__div +
                                  (((hypre__mod) <
                                    (hypre__block)) ? (hypre__mod)
                                   : (hypre__block));
                                hypre__nx =
                                  hypre__div +
                                  ((hypre__mod > hypre__block) ? 1 : 0);
                              }
                            else if (hypre__dir == 1)
                              {
                                loopj =
                                  hypre__block * hypre__div +
                                  (((hypre__mod) <
                                    (hypre__block)) ? (hypre__mod)
                                   : (hypre__block));
                                hypre__ny =
                                  hypre__div +
                                  ((hypre__mod > hypre__block) ? 1 : 0);
                              }
                            else if (hypre__dir == 2)
                              {
                                loopk =
                                  hypre__block * hypre__div +
                                  (((hypre__mod) <
                                    (hypre__block)) ? (hypre__mod)
                                   : (hypre__block));
                                hypre__nz =
                                  hypre__div +
                                  ((hypre__mod > hypre__block) ? 1 : 0);
                              }
                          };
                        Ai =
                          hypre__i1start + loopi * hypre__sx1 +
                          loopj * hypre__sy1 + loopk * hypre__sz1;
                        xi =
                          hypre__i2start + loopi * hypre__sx2 +
                          loopj * hypre__sy2 + loopk * hypre__sz2;
                        ri =
                          hypre__i3start + loopi * hypre__sx3 +
                          loopj * hypre__sy3 + loopk * hypre__sz3;

                          //begin of the loop
#if 0                          //
                        for (loopk = 0; loopk < hypre__nz; loopk++)
                          {
                            for (loopj = 0; loopj < hypre__ny; loopj++)
                              {
                                for (loopi = 0; loopi < hypre__nx; loopi++)
                                  {
                                    {
                                      rp[ri] -= Ap[Ai] * xp[xi];
                                    }
                                    Ai += hypre__sx1;
                                    xi += hypre__sx2;
                                    ri += hypre__sx3;
                                  }
                                Ai += hypre__sy1 - hypre__nx * hypre__sx1;
                                xi += hypre__sy2 - hypre__nx * hypre__sx2;
                                ri += hypre__sy3 - hypre__nx * hypre__sx3;
                              }
                            Ai += hypre__sz1 - hypre__ny * hypre__sy1;
                            xi += hypre__sz2 - hypre__ny * hypre__sy2;
                            ri += hypre__sz3 - hypre__ny * hypre__sy3;
                          } // end of the loop
#else

#if BLCR_CHECKPOINTING
                        // Only checkpoint it at the first occurrance.
                        if (g_checkpoint_flag == 0)
                        {
                          int err;
                          cr_checkpoint_args_t cr_args;  
                          cr_checkpoint_handle_t cr_handle;
                          cr_initialize_checkpoint_args_t(&cr_args);
                          cr_args.cr_scope = CR_SCOPE_PROC;// a process
                          cr_args.cr_target = 0; //self
                          cr_args.cr_signal = SIGKILL; // kill after checkpointing
                          cr_args.cr_fd = open("dump.yy", O_WRONLY|O_CREAT|O_LARGEFILE, 0400);
                          if (cr_args.cr_fd < 0) {
                              printf("Error: cannot open file for checkpoiting context\n");
                              abort();
                          }

                          g_checkpoint_flag ++;
                          printf("Checkpoiting: starting here ..\n");

                          err = cr_request_checkpoint(&cr_args, &cr_handle);
                          if (err < 0) {
                            printf("cannot request checkpoining! err=%d\n",err);
                            abort();
                          }
                          // block until the request is served
                          cr_enter_cs(cr);
                          cr_leave_cs(cr);

                          printf("Checkpoiting: restarting here ..\n");
                        }
#endif

                           OUT__1__6755__(&Ai,&xi,&ri,&Ap,&xp,&rp,&loopi,&loopj,&loopk,&hypre__sx1,&hypre__sy1,&hypre__sz1,&hypre__sx2,&hypre__sy2,&hypre__sz2,&hypre__sx3,&hypre__sy3,&hypre__sz3,&hypre__nx,&hypre__ny,&hypre__nz);

#endif
                      }
                  };
HYPRE_Int
hypre_SMGResidual( void               *residual_vdata,
                   hypre_StructMatrix *A,
                   hypre_StructVector *x,
                   hypre_StructVector *b,
                   hypre_StructVector *r              )
{
   HYPRE_Int ierr;

   hypre_SMGResidualData  *residual_data = residual_vdata;

   hypre_IndexRef          base_stride = (residual_data -> base_stride);
   hypre_BoxArray         *base_points = (residual_data -> base_points);
   hypre_ComputePkg       *compute_pkg = (residual_data -> compute_pkg);

   hypre_CommHandle       *comm_handle;
                       
   hypre_BoxArrayArray    *compute_box_aa;
   hypre_BoxArray         *compute_box_a;
   hypre_Box              *compute_box;
                       
   hypre_Box              *A_data_box;
   hypre_Box              *x_data_box;
   hypre_Box              *b_data_box;
   hypre_Box              *r_data_box;
                       
   HYPRE_Int               Ai;
   HYPRE_Int               xi;
   HYPRE_Int               bi;
   HYPRE_Int               ri;
                         
   double                 *Ap0;
   double                 *xp0;
   double                 *bp;
   double                 *rp;
                       
   hypre_Index             loop_size;
   hypre_IndexRef          start;
                       
   hypre_StructStencil    *stencil;
   hypre_Index            *stencil_shape;
   HYPRE_Int               stencil_size;

   HYPRE_Int               compute_i, i, j, si;
   HYPRE_Int               loopi, loopj, loopk;

   double            *Ap1, *Ap2;
   double            *Ap3, *Ap4;
   double            *Ap5, *Ap6;
   double            *Ap7, *Ap8, *Ap9;
   double            *Ap10, *Ap11, *Ap12, *Ap13, *Ap14;
   double            *Ap15, *Ap16, *Ap17, *Ap18;
   double            *Ap19, *Ap20, *Ap21, *Ap22, *Ap23, *Ap24, *Ap25, *Ap26;
   double            *xp1, *xp2;
   double            *xp3, *xp4;
   double            *xp5, *xp6;
   double            *xp7, *xp8, *xp9;
   double            *xp10, *xp11, *xp12, *xp13, *xp14;
   double            *xp15, *xp16, *xp17, *xp18;
   double            *xp19, *xp20, *xp21, *xp22, *xp23, *xp24, *xp25, *xp26;

   hypre_BeginTiming(residual_data -> time_index);

   /*-----------------------------------------------------------------------
    * Compute residual r = b - Ax
    *-----------------------------------------------------------------------*/

   stencil       = hypre_StructMatrixStencil(A);
   stencil_shape = hypre_StructStencilShape(stencil);
   stencil_size  = hypre_StructStencilSize(stencil);

   for (compute_i = 0; compute_i < 2; compute_i++)
   {
      switch(compute_i)
      {
         case 0:
         {
            xp0 = hypre_StructVectorData(x);
            hypre_InitializeIndtComputations(compute_pkg, xp0, &comm_handle);
            compute_box_aa = hypre_ComputePkgIndtBoxes(compute_pkg);

            /*----------------------------------------
             * Copy b into r
             *----------------------------------------*/

            compute_box_a = base_points;
            hypre_ForBoxI(i, compute_box_a)
               {
                  compute_box = hypre_BoxArrayBox(compute_box_a, i);
                  start = hypre_BoxIMin(compute_box);

                  b_data_box =
                     hypre_BoxArrayBox(hypre_StructVectorDataSpace(b), i);
                  r_data_box =
                     hypre_BoxArrayBox(hypre_StructVectorDataSpace(r), i);

                  bp = hypre_StructVectorBoxData(b, i);
                  rp = hypre_StructVectorBoxData(r, i);

                  hypre_BoxGetStrideSize(compute_box, base_stride, loop_size);
                  hypre_BoxLoop2Begin(loop_size,
                                      b_data_box, start, base_stride, bi,
                                      r_data_box, start, base_stride, ri);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,bi,ri
#include "hypre_box_smp_forloop.h"
                  hypre_BoxLoop2For(loopi, loopj, loopk, bi, ri)
                     {
                        rp[ri] = bp[bi];
                     }
                  hypre_BoxLoop2End(bi, ri);
               }
         }
         break;

         case 1:
         {
            hypre_FinalizeIndtComputations(comm_handle);
            compute_box_aa = hypre_ComputePkgDeptBoxes(compute_pkg);
         }
         break;
      }

      /*--------------------------------------------------------------------
       * Compute r -= A*x
       *--------------------------------------------------------------------*/

      hypre_ForBoxArrayI(i, compute_box_aa)
         {
            compute_box_a = hypre_BoxArrayArrayBoxArray(compute_box_aa, i);

            A_data_box = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(A), i);
            x_data_box = hypre_BoxArrayBox(hypre_StructVectorDataSpace(x), i);
            r_data_box = hypre_BoxArrayBox(hypre_StructVectorDataSpace(r), i);

            rp = hypre_StructVectorBoxData(r, i);

            /*--------------------------------------------------------------
             * Switch statement to direct control (based on stencil size) to
             * code to get pointers and offsets fo A and x.
             *--------------------------------------------------------------*/

            switch (stencil_size)
            {
               case 1:

               Ap0 = hypre_StructMatrixBoxData(A, i, 0);
               xp0 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[0]);

               break;

               case 3:

               Ap0 = hypre_StructMatrixBoxData(A, i, 0);
               Ap1 = hypre_StructMatrixBoxData(A, i, 1);
               Ap2 = hypre_StructMatrixBoxData(A, i, 2);

               xp0 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[0]);
               xp1 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[1]);
               xp2 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[2]);

               break;

               case 5:

               Ap0 = hypre_StructMatrixBoxData(A, i, 0);
               Ap1 = hypre_StructMatrixBoxData(A, i, 1);
               Ap2 = hypre_StructMatrixBoxData(A, i, 2);
               Ap3 = hypre_StructMatrixBoxData(A, i, 3);
               Ap4 = hypre_StructMatrixBoxData(A, i, 4);

               xp0 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[0]);
               xp1 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[1]);
               xp2 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[2]);
               xp3 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[3]);
               xp4 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[4]);

               break;

               case 7:

               Ap0 = hypre_StructMatrixBoxData(A, i, 0);
               Ap1 = hypre_StructMatrixBoxData(A, i, 1);
               Ap2 = hypre_StructMatrixBoxData(A, i, 2);
               Ap3 = hypre_StructMatrixBoxData(A, i, 3);
               Ap4 = hypre_StructMatrixBoxData(A, i, 4);
               Ap5 = hypre_StructMatrixBoxData(A, i, 5);
               Ap6 = hypre_StructMatrixBoxData(A, i, 6);

               xp0 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[0]);
               xp1 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[1]);
               xp2 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[2]);
               xp3 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[3]);
               xp4 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[4]);
               xp5 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[5]);
               xp6 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[6]);

               break;

               case 9:

               Ap0 = hypre_StructMatrixBoxData(A, i, 0);
               Ap1 = hypre_StructMatrixBoxData(A, i, 1);
               Ap2 = hypre_StructMatrixBoxData(A, i, 2);
               Ap3 = hypre_StructMatrixBoxData(A, i, 3);
               Ap4 = hypre_StructMatrixBoxData(A, i, 4);
               Ap5 = hypre_StructMatrixBoxData(A, i, 5);
               Ap6 = hypre_StructMatrixBoxData(A, i, 6);
               Ap7 = hypre_StructMatrixBoxData(A, i, 7);
               Ap8 = hypre_StructMatrixBoxData(A, i, 8);

               xp0 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[0]);
               xp1 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[1]);
               xp2 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[2]);
               xp3 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[3]);
               xp4 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[4]);
               xp5 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[5]);
               xp6 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[6]);
               xp7 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[7]);
               xp8 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[8]);

               break;

               case 15:

               Ap0 = hypre_StructMatrixBoxData(A, i, 0);
               Ap1 = hypre_StructMatrixBoxData(A, i, 1);
               Ap2 = hypre_StructMatrixBoxData(A, i, 2);
               Ap3 = hypre_StructMatrixBoxData(A, i, 3);
               Ap4 = hypre_StructMatrixBoxData(A, i, 4);
               Ap5 = hypre_StructMatrixBoxData(A, i, 5);
               Ap6 = hypre_StructMatrixBoxData(A, i, 6);
               Ap7 = hypre_StructMatrixBoxData(A, i, 7);
               Ap8 = hypre_StructMatrixBoxData(A, i, 8);
               Ap9 = hypre_StructMatrixBoxData(A, i, 9);
               Ap10 = hypre_StructMatrixBoxData(A, i, 10);
               Ap11 = hypre_StructMatrixBoxData(A, i, 11);
               Ap12 = hypre_StructMatrixBoxData(A, i, 12);
               Ap13 = hypre_StructMatrixBoxData(A, i, 13);
               Ap14 = hypre_StructMatrixBoxData(A, i, 14);

               xp0 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[0]);
               xp1 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[1]);
               xp2 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[2]);
               xp3 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[3]);
               xp4 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[4]);
               xp5 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[5]);
               xp6 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[6]);
               xp7 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[7]);
               xp8 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[8]);
               xp9 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[9]);
               xp10 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[10]);
               xp11 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[11]);
               xp12 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[12]);
               xp13 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[13]);
               xp14 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[14]);

               break;

               case 19:

               Ap0 = hypre_StructMatrixBoxData(A, i, 0);
               Ap1 = hypre_StructMatrixBoxData(A, i, 1);
               Ap2 = hypre_StructMatrixBoxData(A, i, 2);
               Ap3 = hypre_StructMatrixBoxData(A, i, 3);
               Ap4 = hypre_StructMatrixBoxData(A, i, 4);
               Ap5 = hypre_StructMatrixBoxData(A, i, 5);
               Ap6 = hypre_StructMatrixBoxData(A, i, 6);
               Ap7 = hypre_StructMatrixBoxData(A, i, 7);
               Ap8 = hypre_StructMatrixBoxData(A, i, 8);
               Ap9 = hypre_StructMatrixBoxData(A, i, 9);
               Ap10 = hypre_StructMatrixBoxData(A, i, 10);
               Ap11 = hypre_StructMatrixBoxData(A, i, 11);
               Ap12 = hypre_StructMatrixBoxData(A, i, 12);
               Ap13 = hypre_StructMatrixBoxData(A, i, 13);
               Ap14 = hypre_StructMatrixBoxData(A, i, 14);
               Ap15 = hypre_StructMatrixBoxData(A, i, 15);
               Ap16 = hypre_StructMatrixBoxData(A, i, 16);
               Ap17 = hypre_StructMatrixBoxData(A, i, 17);
               Ap18 = hypre_StructMatrixBoxData(A, i, 18);

               xp0 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[0]);
               xp1 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[1]);
               xp2 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[2]);
               xp3 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[3]);
               xp4 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[4]);
               xp5 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[5]);
               xp6 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[6]);
               xp7 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[7]);
               xp8 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[8]);
               xp9 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[9]);
               xp10 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[10]);
               xp11 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[11]);
               xp12 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[12]);
               xp13 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[13]);
               xp14 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[14]);
               xp15 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[15]);
               xp16 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[16]);
               xp17 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[17]);
               xp18 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[18]);

               break;

               case 27:

               Ap0 = hypre_StructMatrixBoxData(A, i, 0);
               Ap1 = hypre_StructMatrixBoxData(A, i, 1);
               Ap2 = hypre_StructMatrixBoxData(A, i, 2);
               Ap3 = hypre_StructMatrixBoxData(A, i, 3);
               Ap4 = hypre_StructMatrixBoxData(A, i, 4);
               Ap5 = hypre_StructMatrixBoxData(A, i, 5);
               Ap6 = hypre_StructMatrixBoxData(A, i, 6);
               Ap7 = hypre_StructMatrixBoxData(A, i, 7);
               Ap8 = hypre_StructMatrixBoxData(A, i, 8);
               Ap9 = hypre_StructMatrixBoxData(A, i, 9);
               Ap10 = hypre_StructMatrixBoxData(A, i, 10);
               Ap11 = hypre_StructMatrixBoxData(A, i, 11);
               Ap12 = hypre_StructMatrixBoxData(A, i, 12);
               Ap13 = hypre_StructMatrixBoxData(A, i, 13);
               Ap14 = hypre_StructMatrixBoxData(A, i, 14);
               Ap15 = hypre_StructMatrixBoxData(A, i, 15);
               Ap16 = hypre_StructMatrixBoxData(A, i, 16);
               Ap17 = hypre_StructMatrixBoxData(A, i, 17);
               Ap18 = hypre_StructMatrixBoxData(A, i, 18);
               Ap19 = hypre_StructMatrixBoxData(A, i, 19);
               Ap20 = hypre_StructMatrixBoxData(A, i, 20);
               Ap21 = hypre_StructMatrixBoxData(A, i, 21);
               Ap22 = hypre_StructMatrixBoxData(A, i, 22);
               Ap23 = hypre_StructMatrixBoxData(A, i, 23);
               Ap24 = hypre_StructMatrixBoxData(A, i, 24);
               Ap25 = hypre_StructMatrixBoxData(A, i, 25);
               Ap26 = hypre_StructMatrixBoxData(A, i, 26);

               xp0 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[0]);
               xp1 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[1]);
               xp2 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[2]);
               xp3 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[3]);
               xp4 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[4]);
               xp5 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[5]);
               xp6 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[6]);
               xp7 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[7]);
               xp8 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[8]);
               xp9 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[9]);
               xp10 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[10]);
               xp11 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[11]);
               xp12 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[12]);
               xp13 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[13]);
               xp14 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[14]);
               xp15 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[15]);
               xp16 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[16]);
               xp17 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[17]);
               xp18 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[18]);
               xp19 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[19]);
               xp20 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[20]);
               xp21 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[21]);
               xp22 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[22]);
               xp23 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[23]);
               xp24 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[24]);
               xp25 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[25]);
               xp26 = hypre_StructVectorBoxData(x, i) +
                  hypre_BoxOffsetDistance(x_data_box, stencil_shape[26]);

               break;

               default:
               ;
            }

            hypre_ForBoxI(j, compute_box_a)
               {
                  compute_box = hypre_BoxArrayBox(compute_box_a, j);

                  start  = hypre_BoxIMin(compute_box);

                  /*------------------------------------------------------
                   * Switch statement to direct control to appropriate
                   * box loop depending on stencil size
                   *------------------------------------------------------*/

                  switch (stencil_size)
                  {

                     case 1:
   
                     hypre_BoxGetStrideSize(compute_box, base_stride, loop_size);
                     hypre_BoxLoop3Begin(loop_size,
                                         A_data_box, start, base_stride, Ai,
                                         x_data_box, start, base_stride, xi,
                                         r_data_box, start, base_stride, ri);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,Ai,xi,ri
#include "hypre_box_smp_forloop.h"
                     hypre_BoxLoop3For(loopi, loopj, loopk, Ai, xi, ri)
                        {

                           rp[ri] = rp[ri]
                              - Ap0[Ai] * xp0[xi];

                        }
                     hypre_BoxLoop3End(Ai, xi, ri);

                     break;

                     case 3:

                     hypre_BoxGetStrideSize(compute_box, base_stride, loop_size);
                     hypre_BoxLoop3Begin(loop_size,
                                         A_data_box, start, base_stride, Ai,
                                         x_data_box, start, base_stride, xi,
                                         r_data_box, start, base_stride, ri);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,Ai,xi,ri
#include "hypre_box_smp_forloop.h"
                     hypre_BoxLoop3For(loopi, loopj, loopk, Ai, xi, ri)
                        {
 
                           rp[ri] = rp[ri]
                              - Ap0[Ai] * xp0[xi]
                              - Ap1[Ai] * xp1[xi]
                              - Ap2[Ai] * xp2[xi];

                        }
int
hypre_SemiRestrict( void               *restrict_vdata,
                    hypre_StructMatrix *R,
                    hypre_StructVector *r,
                    hypre_StructVector *rc             )
{
   int ierr = 0;

   hypre_SemiRestrictData *restrict_data = (hypre_SemiRestrictData *)restrict_vdata;

   int                     R_stored_as_transpose;
   hypre_ComputePkg       *compute_pkg;
   hypre_IndexRef          cindex;
   hypre_IndexRef          stride;

   hypre_StructGrid       *fgrid;
   int                    *fgrid_ids;
   hypre_StructGrid       *cgrid;
   hypre_BoxArray         *cgrid_boxes;
   int                    *cgrid_ids;

   hypre_CommHandle       *comm_handle;
                       
   hypre_BoxArrayArray    *compute_box_aa;
   hypre_BoxArray         *compute_box_a;
   hypre_Box              *compute_box;
                       
   hypre_Box              *R_dbox;
   hypre_Box              *r_dbox;
   hypre_Box              *rc_dbox;
                       
   int                     Ri;
   int                     ri;
   int                     rci;
                         
   double                 *Rp0, *Rp1;
   double                 *rp, *rp0, *rp1;
   double                 *rcp;
                       
   hypre_Index             loop_size;
   hypre_IndexRef          start;
   hypre_Index             startc;
   hypre_Index             stridec;
                       
   hypre_StructStencil    *stencil;
   hypre_Index            *stencil_shape;

   int                     compute_i, fi, ci, j;
   int                     loopi, loopj, loopk;

   /*-----------------------------------------------------------------------
    * Initialize some things.
    *-----------------------------------------------------------------------*/

   hypre_BeginTiming(restrict_data -> time_index);

   R_stored_as_transpose = (restrict_data -> R_stored_as_transpose);
   compute_pkg   = (restrict_data -> compute_pkg);
   cindex        = (restrict_data -> cindex);
   stride        = (restrict_data -> stride);

   stencil       = hypre_StructMatrixStencil(R);
   stencil_shape = hypre_StructStencilShape(stencil);

   hypre_SetIndex(stridec, 1, 1, 1);

   /*--------------------------------------------------------------------
    * Restrict the residual.
    *--------------------------------------------------------------------*/

   fgrid = hypre_StructVectorGrid(r);
   fgrid_ids = hypre_StructGridIDs(fgrid);
   cgrid = hypre_StructVectorGrid(rc);
   cgrid_boxes = hypre_StructGridBoxes(cgrid);
   cgrid_ids = hypre_StructGridIDs(cgrid);

   for (compute_i = 0; compute_i < 2; compute_i++)
   {
      switch(compute_i)
      {
         case 0:
         {
            rp = hypre_StructVectorData(r);
            hypre_InitializeIndtComputations(compute_pkg, rp, &comm_handle);
            compute_box_aa = hypre_ComputePkgIndtBoxes(compute_pkg);
         }
         break;

         case 1:
         {
            hypre_FinalizeIndtComputations(comm_handle);
            compute_box_aa = hypre_ComputePkgDeptBoxes(compute_pkg);
         }
         break;
      }

      fi = 0;
      hypre_ForBoxArrayI(ci, cgrid_boxes)
         {
            while (fgrid_ids[fi] != cgrid_ids[ci])
            {
               fi++;
            }

            compute_box_a = hypre_BoxArrayArrayBoxArray(compute_box_aa, fi);

            R_dbox  = hypre_BoxArrayBox(hypre_StructMatrixDataSpace(R),  fi);
            r_dbox  = hypre_BoxArrayBox(hypre_StructVectorDataSpace(r),  fi);
            rc_dbox = hypre_BoxArrayBox(hypre_StructVectorDataSpace(rc), ci);

            if (R_stored_as_transpose)
            {
               Rp0 = hypre_StructMatrixBoxData(R, fi, 1) -
                  hypre_BoxOffsetDistance(R_dbox, stencil_shape[1]);
               Rp1 = hypre_StructMatrixBoxData(R, fi, 0);
            }
            else
            {
               Rp0 = hypre_StructMatrixBoxData(R, fi, 0);
               Rp1 = hypre_StructMatrixBoxData(R, fi, 1);
            }
            rp  = hypre_StructVectorBoxData(r, fi);
            rp0 = rp + hypre_BoxOffsetDistance(r_dbox, stencil_shape[0]);
            rp1 = rp + hypre_BoxOffsetDistance(r_dbox, stencil_shape[1]);
            rcp = hypre_StructVectorBoxData(rc, ci);

            hypre_ForBoxI(j, compute_box_a)
               {
                  compute_box = hypre_BoxArrayBox(compute_box_a, j);

                  start  = hypre_BoxIMin(compute_box);
                  hypre_StructMapFineToCoarse(start, cindex, stride, startc);

                  hypre_BoxGetStrideSize(compute_box, stride, loop_size);
                  hypre_BoxLoop3Begin(loop_size,
                                      R_dbox,  startc, stridec, Ri,
                                      r_dbox,  start,  stride,  ri,
                                      rc_dbox, startc, stridec, rci);
#define HYPRE_BOX_SMP_PRIVATE loopk,loopi,loopj,Ri,ri,rci
#include "hypre_box_smp_forloop.h"
                  hypre_BoxLoop3For(loopi, loopj, loopk, Ri, ri, rci)
                     {
                        rcp[rci] = rp[ri] + (Rp0[Ri] * rp0[ri] +
                                             Rp1[Ri] * rp1[ri]);
                     }
                  hypre_BoxLoop3End(Ri, ri, rci);
               }
         }