Esempio n. 1
0
HYPRE_Int
hypre_FillResponseParToVectorAll( void       *p_recv_contact_buf, 
                                  HYPRE_Int   contact_size,
                                  HYPRE_Int   contact_proc,
                                  void       *ro, 
                                  MPI_Comm    comm,
                                  void      **p_send_response_buf, 
                                  HYPRE_Int  *response_message_size )
{
   HYPRE_Int     myid;
   HYPRE_Int     i, index, count, elength;

   HYPRE_Int    *recv_contact_buf = (HYPRE_Int * ) p_recv_contact_buf;

   hypre_DataExchangeResponse  *response_obj = ro;  

   hypre_ProcListElements      *send_proc_obj = response_obj->data2;   

   hypre_MPI_Comm_rank(comm, &myid );

   /*check to see if we need to allocate more space in send_proc_obj for ids*/
   if (send_proc_obj->length == send_proc_obj->storage_length)
   {
      send_proc_obj->storage_length +=10; /*add space for 10 more processors*/
      send_proc_obj->id = hypre_TReAlloc(send_proc_obj->id,HYPRE_Int, 
                                         send_proc_obj->storage_length);
      send_proc_obj->vec_starts =
         hypre_TReAlloc(send_proc_obj->vec_starts,HYPRE_Int,
                        send_proc_obj->storage_length + 1);
   }
  
   /*initialize*/ 
   count = send_proc_obj->length;
   index = send_proc_obj->vec_starts[count]; /*this is the number of elements*/

   /*send proc*/ 
   send_proc_obj->id[count] = contact_proc; 

   /*do we need more storage for the elements?*/
   if (send_proc_obj->element_storage_length < index + contact_size)
   {
      elength = hypre_max(contact_size, 10);   
      elength += index;
      send_proc_obj->elements = hypre_TReAlloc(send_proc_obj->elements, 
                                               HYPRE_Int, elength);
      send_proc_obj->element_storage_length = elength; 
   }
   /*populate send_proc_obj*/
   for (i=0; i< contact_size; i++) 
   { 
      send_proc_obj->elements[index++] = recv_contact_buf[i];
   }
   send_proc_obj->vec_starts[count+1] = index;
   send_proc_obj->length++;

   /*output - no message to return (confirmation) */
   *response_message_size = 0; 
  
   return hypre_error_flag;
}
Esempio n. 2
0
File: box.c Progetto: tpatki/sequoia
int
hypre_BoxArraySetSize( hypre_BoxArray  *box_array,
                       int              size      )
{
    int  ierr  = 0;
    int  alloc_size;

    alloc_size = hypre_BoxArrayAllocSize(box_array);

    if (size > alloc_size)
    {
        alloc_size = size + hypre_BoxArrayExcess;

        hypre_BoxArrayBoxes(box_array) =
            hypre_TReAlloc(hypre_BoxArrayBoxes(box_array),
                           hypre_Box, alloc_size);

        hypre_BoxArrayAllocSize(box_array) = alloc_size;
    }

    hypre_BoxArraySize(box_array) = size;

    return ierr;
}
Esempio n. 3
0
/******************************************************************************
 *
 * hypre_IJVectorAddToValuesPar
 *
 * adds to a potentially noncontiguous set of IJVectorPar components
 *
 *****************************************************************************/
HYPRE_Int
hypre_IJVectorAddToValuesPar(hypre_IJVector *vector,
                             HYPRE_Int             num_values,
                             const HYPRE_Int      *indices,
                             const double   *values      )
{
   HYPRE_Int my_id;
   HYPRE_Int i, j, vec_start, vec_stop;
   double *data;
   HYPRE_Int print_level = hypre_IJVectorPrintLevel(vector);

   HYPRE_Int *IJpartitioning = hypre_IJVectorPartitioning(vector);
   hypre_ParVector *par_vector = hypre_IJVectorObject(vector);
   hypre_AuxParVector *aux_vector = hypre_IJVectorTranslator(vector);
   MPI_Comm comm = hypre_IJVectorComm(vector);
   hypre_Vector *local_vector = hypre_ParVectorLocalVector(par_vector);

/* If no components are to be retrieved, perform no checking and return */
   if (num_values < 1) return 0;

   hypre_MPI_Comm_rank(comm, &my_id);

/* If par_vector == NULL or partitioning == NULL or local_vector == NULL 
   let user know of catastrophe and exit */

   if (!par_vector)
   {
      if (print_level)
      {
         hypre_printf("par_vector == NULL -- ");
         hypre_printf("hypre_IJVectorAddToValuesPar\n");
         hypre_printf("**** Vector storage is either unallocated or orphaned ****\n");
      }
      hypre_error_in_arg(1);
   }
   if (!IJpartitioning)
   {
      if (print_level)
      {
         hypre_printf("IJpartitioning == NULL -- ");
         hypre_printf("hypre_IJVectorAddToValuesPar\n");
         hypre_printf("**** IJVector partitioning is either unallocated or orphaned ****\n");
      }
      hypre_error_in_arg(1);
   }
   if (!local_vector)
   {
      if (print_level)
      {
         hypre_printf("local_vector == NULL -- ");
         hypre_printf("hypre_IJVectorAddToValuesPar\n");
         hypre_printf("**** Vector local data is either unallocated or orphaned ****\n");
      }
      hypre_error_in_arg(1);
   }

#ifdef HYPRE_NO_GLOBAL_PARTITION
   vec_start = IJpartitioning[0];
   vec_stop  = IJpartitioning[1]-1;
#else
   vec_start = IJpartitioning[my_id];
   vec_stop  = IJpartitioning[my_id+1]-1;
#endif

   if (vec_start > vec_stop) 
   {
      if (print_level)
      {
         hypre_printf("vec_start > vec_stop -- ");
         hypre_printf("hypre_IJVectorAddToValuesPar\n");
         hypre_printf("**** This vector partitioning should not occur ****\n");
      }
      hypre_error_in_arg(1);
   }

/* Determine whether indices points to local indices only,
   and if not, store indices and values into auxiliary vector structure
   If indices == NULL, assume that num_values components are to be
   set in a block starting at vec_start. 
   NOTE: If indices == NULL off processor values are ignored!!! */

   /* if (indices)
   {
      for (i = 0; i < num_values; i++)
      { 	
	ierr += (indices[i] <  vec_start);
        ierr += (indices[i] >= vec_stop);
      }
   }

   if (ierr)
   {
      hypre_printf("indices beyond local range -- ");
      hypre_printf("hypre_IJVectorAddToValuesPar\n");
      hypre_printf("**** Indices specified are unusable ****\n");
      exit(1);
   } */
    
   data = hypre_VectorData(local_vector);

   if (indices)
   {
      HYPRE_Int current_num_elmts
                = hypre_AuxParVectorCurrentNumElmts(aux_vector);
      HYPRE_Int max_off_proc_elmts
                = hypre_AuxParVectorMaxOffProcElmts(aux_vector);
      HYPRE_Int *off_proc_i = hypre_AuxParVectorOffProcI(aux_vector);
      double *off_proc_data = hypre_AuxParVectorOffProcData(aux_vector);

      for (j = 0; j < num_values; j++)
      {
	 i = indices[j];
	 if (i < vec_start || i > vec_stop)
         {
	 /* if elements outside processor boundaries, store in off processor
		stash */
	    if (!max_off_proc_elmts)
            {
               max_off_proc_elmts = 100;
               hypre_AuxParVectorMaxOffProcElmts(aux_vector) =
                        max_off_proc_elmts;
               hypre_AuxParVectorOffProcI(aux_vector)
                        = hypre_CTAlloc(HYPRE_Int,max_off_proc_elmts);
               hypre_AuxParVectorOffProcData(aux_vector)
                        = hypre_CTAlloc(double,max_off_proc_elmts);
               off_proc_i = hypre_AuxParVectorOffProcI(aux_vector);
               off_proc_data = hypre_AuxParVectorOffProcData(aux_vector);
            }
            else if (current_num_elmts + 1 > max_off_proc_elmts)
            {
               max_off_proc_elmts += 10;
               off_proc_i = hypre_TReAlloc(off_proc_i,HYPRE_Int,max_off_proc_elmts);
               off_proc_data = hypre_TReAlloc(off_proc_data,double,
                                max_off_proc_elmts);
               hypre_AuxParVectorMaxOffProcElmts(aux_vector)
                        = max_off_proc_elmts;
               hypre_AuxParVectorOffProcI(aux_vector) = off_proc_i;
               hypre_AuxParVectorOffProcData(aux_vector) = off_proc_data;
            }
            off_proc_i[current_num_elmts] = i;
            off_proc_data[current_num_elmts++] = values[j];
            hypre_AuxParVectorCurrentNumElmts(aux_vector)=current_num_elmts;
         }
Esempio n. 4
0
HYPRE_Int hypre_DataExchangeList(HYPRE_Int num_contacts, 
                                 HYPRE_Int *contact_proc_list,
                                 void *contact_send_buf, 
                                 HYPRE_Int *contact_send_buf_starts,
                                 HYPRE_Int contact_obj_size, 
                                 HYPRE_Int response_obj_size,
                                 hypre_DataExchangeResponse *response_obj,
                                 HYPRE_Int max_response_size, 
                                 HYPRE_Int rnum, MPI_Comm comm,
                                 void **p_response_recv_buf, 
                                 HYPRE_Int **p_response_recv_buf_starts)
{
   /*-------------------------------------------
    *  parameters: 
    *   
    *    num_contacts              = how many procs to contact
    *    contact_proc_list         = list of processors to contact 
    *    contact_send_buf          = array of data to send
    *    contact_send_buf_starts   = index for contact_send_buf corresponding to 
    *                                contact_proc_list
    *    contact_obj_size          = sizeof() one obj in contact list  
   
    *    response_obj_size          = sizeof() one obj in response_recv_buf 
    *    response_obj              = this will give us the function we need to 
    *                                fill the reponse as well as 
    *                                any data we might need to accomplish that 
    *    max_response_size         = max size of a single response expected (do NOT
    *                                need to be an absolute upper bound)  
    *    rnum                      = two consequentive exchanges should have different 
    *                                rnums. Alternate rnum = 1 
    *                                and rnum=2  - these flags will be even (so odd
    *                                numbered tags could be used in calling code) 
    *    p_response_recv_buf       = where to receive the reponses - will be allocated 
    *                                in this function  
    *    p_response_recv_buf_starts  = index of p_response_buf corresponding to 
    *                                contact_buf_list - will be allocated here  

    *-------------------------------------------*/

   HYPRE_Int  num_procs, myid;
   HYPRE_Int  i;
   HYPRE_Int  terminate, responses_complete;
   HYPRE_Int  children_complete;
   HYPRE_Int  contact_flag;
   HYPRE_Int  proc;
   HYPRE_Int  contact_size;

   HYPRE_Int  size, post_size, copy_size;
   HYPRE_Int  total_size, count;      

   void *start_ptr = NULL, *index_ptr=NULL;
   HYPRE_Int  *int_ptr=NULL;  

   void *response_recv_buf = NULL;
   void *send_response_buf = NULL;

   HYPRE_Int  *response_recv_buf_starts = NULL;
   void *initial_recv_buf = NULL;  

   void *recv_contact_buf = NULL;
   HYPRE_Int  recv_contact_buf_size = 0;
   
   HYPRE_Int  response_message_size = 0;

   HYPRE_Int  overhead;
 
   HYPRE_Int  max_response_size_bytes;
   
   HYPRE_Int  max_response_total_bytes;
   
   void **post_array = NULL;  /*this must be set to null or realloc will crash */
   HYPRE_Int  post_array_storage = 0;
   HYPRE_Int  post_array_size = 0;
   HYPRE_Int   num_post_recvs =0;
   
   void **contact_ptrs = NULL, **response_ptrs=NULL, **post_ptrs=NULL;
   
   hypre_BinaryTree tree;

   hypre_MPI_Request *response_requests, *contact_requests;
   hypre_MPI_Status  *response_statuses, *contact_statuses;
   
   hypre_MPI_Request  *post_send_requests = NULL, *post_recv_requests = NULL;
   hypre_MPI_Status   *post_send_statuses = NULL, *post_recv_statuses = NULL;

   hypre_MPI_Request *term_requests, term_request1, request_parent;
   hypre_MPI_Status  *term_statuses, term_status1, status_parent;
   hypre_MPI_Status  status, fill_status;

   const HYPRE_Int contact_tag = 1000*rnum;
   const HYPRE_Int response_tag = 1002*rnum;
   const HYPRE_Int term_tag =  1004*rnum;  
   const HYPRE_Int post_tag = 1006*rnum;
   
   hypre_MPI_Comm_size(comm, &num_procs );
   hypre_MPI_Comm_rank(comm, &myid );

   /* ---------initializations ----------------*/ 

   /* if the response_obj_size or contact_obj_size is 0, set to sizeof(HYPRE_Int) */
   if (!response_obj_size) response_obj_size = sizeof(HYPRE_Int);
   if (!contact_obj_size) contact_obj_size = sizeof(HYPRE_Int);

   max_response_size_bytes = max_response_size*response_obj_size;
 
                   
   /* pre-allocate the max space for responding to contacts */
   overhead = ceil((HYPRE_Real) sizeof(HYPRE_Int)/response_obj_size); /*for appending an integer*/
   
   max_response_total_bytes = (max_response_size+overhead)*response_obj_size;

   response_obj->send_response_overhead = overhead;
   response_obj->send_response_storage = max_response_size;

   /*send_response_buf = hypre_MAlloc(max_response_total_bytes);*/
   send_response_buf = hypre_CAlloc(max_response_size+overhead, response_obj_size);
      
   /*allocate space for inital recv array for the responses - give each processor
     size max_response_size */

   initial_recv_buf = hypre_MAlloc(max_response_total_bytes*num_contacts);
   response_recv_buf_starts =   hypre_CTAlloc(HYPRE_Int, num_contacts+1);  

   contact_ptrs = hypre_TAlloc( void *, num_contacts);
   response_ptrs = hypre_TAlloc(void *, num_contacts);
   
   /*-------------SEND CONTACTS AND POST RECVS FOR RESPONSES---*/

   for (i=0; i<= num_contacts; i++)
   {
      response_recv_buf_starts[i] = i*(max_response_size+overhead);
   }
   
   /* Send "contact" messages to the list of processors and 
      pre-post receives to wait for their response*/

   responses_complete = 1;
   if (num_contacts > 0 )
   {
      responses_complete = 0;
      response_requests = hypre_CTAlloc(hypre_MPI_Request, num_contacts);
      response_statuses = hypre_CTAlloc(hypre_MPI_Status, num_contacts);
      contact_requests = hypre_CTAlloc(hypre_MPI_Request, num_contacts);
      contact_statuses = hypre_CTAlloc(hypre_MPI_Status, num_contacts);

      /* post receives - could be confirmation or data*/
      /* the size to post is max_response_total_bytes*/     
      
      for (i=0; i< num_contacts; i++) 
      {
         /* response_ptrs[i] =  initial_recv_buf + i*max_response_total_bytes ; */
         response_ptrs[i] = (void *)((char *) initial_recv_buf +
                                     i*max_response_total_bytes) ; 

         hypre_MPI_Irecv(response_ptrs[i], max_response_total_bytes,
                         hypre_MPI_BYTE, contact_proc_list[i], 
                         response_tag, comm, &response_requests[i]);
      }

      /* send out contact messages */
      start_ptr = contact_send_buf;
      for (i=0; i< num_contacts; i++) 
      {
         contact_ptrs[i] = start_ptr; 
         size =  contact_send_buf_starts[i+1] - contact_send_buf_starts[i]  ; 
         hypre_MPI_Isend(contact_ptrs[i], size*contact_obj_size,
                         hypre_MPI_BYTE, contact_proc_list[i], 
                         contact_tag, comm, &contact_requests[i]); 
         /*  start_ptr += (size*contact_obj_size); */        
         start_ptr = (void *) ((char *) start_ptr  + (size*contact_obj_size)); 
      }
   }

   /*------------BINARY TREE-----------------------*/

   /*Now let's find out our binary tree information and
     initialize for the termination check sweep */
   terminate = 1; /*indicates whether we can stop probing for contact */
   children_complete = 1;/*indicates whether we have recv. term messages 
                           from our children*/
 
   if (num_procs > 1)
   { 
      hypre_CreateBinaryTree(myid, num_procs, &tree);

      /* we will get a message from all of our children when they 
         have received responses for all of their contacts.  
         So post receives now */
   
      term_requests = hypre_CTAlloc(hypre_MPI_Request, tree.num_child); 
      term_statuses = hypre_CTAlloc(hypre_MPI_Status, tree.num_child);

      for (i=0; i< tree.num_child; i++) 
      {
	 hypre_MPI_Irecv(NULL, 0, HYPRE_MPI_INT, tree.child_id[i], term_tag, comm, 
                         &term_requests[i]);
      }

      terminate = 0;
 
      children_complete = 0;
   }
   else if (num_procs ==1 && num_contacts > 0 ) /* added 11/08 */
   {
      terminate = 0;
   }

   /*---------PROBE LOOP-----------------------------------------*/

   /*Look for incoming contact messages - don't know how many I will get!*/
  
   while (!terminate)
   {
      /* did I receive any contact messages? */
      hypre_MPI_Iprobe(hypre_MPI_ANY_SOURCE, contact_tag, comm,
                       &contact_flag, &status);

      while (contact_flag)
      {
         /* received contacts - from who and what do we do ?*/
         proc = status.hypre_MPI_SOURCE;
         hypre_MPI_Get_count(&status, hypre_MPI_BYTE, &contact_size);

         contact_size = contact_size/contact_obj_size;
         
         /*---------------FILL RESPONSE ------------------------*/

         /*first receive the contact buffer - then call a function
           to determine how to populate the send buffer for the reponse*/

         /* do we have enough space to recv it? */
         if(contact_size > recv_contact_buf_size) 
         {
            recv_contact_buf = hypre_ReAlloc(recv_contact_buf, 
                                             contact_obj_size*contact_size);
            recv_contact_buf_size = contact_size;
         } 

         /* this must be blocking - can't fill recv without the buffer*/         
         hypre_MPI_Recv(recv_contact_buf, contact_size*contact_obj_size,
                        hypre_MPI_BYTE, proc, contact_tag, comm, &fill_status);

         response_obj->fill_response(recv_contact_buf, contact_size, proc,
                                     response_obj, comm, &send_response_buf,
                                     &response_message_size );

         /* we need to append the size of the send obj */
         /* first we copy out any part that may be needed to send later so we don't overwrite */
         post_size = response_message_size - max_response_size; 
         if (post_size > 0) /*we will need to send the extra information later */   
         {
            /*hypre_printf("myid = %d, post_size = %d\n", myid, post_size);*/

            if (post_array_size == post_array_storage)
             
            {
               /* allocate room for more posts  - add 20*/
               post_array_storage += 20;       
               post_array = hypre_TReAlloc(post_array, void *, post_array_storage); 
               post_send_requests =
                  hypre_TReAlloc(post_send_requests, hypre_MPI_Request,
                                 post_array_storage);
            }
            /* allocate space for the data this post only*/
            /* this should not happen often (unless a poor max_size has been chosen)
               - so we will allocate space for the data as needed */
            size = post_size*response_obj_size;
            post_array[post_array_size] =  hypre_MAlloc(size); 
            /* index_ptr =  send_response_buf + max_response_size_bytes */;
            index_ptr = (void *) ((char *) send_response_buf +
                                  max_response_size_bytes);
               
            memcpy(post_array[post_array_size], index_ptr, size);
    
            /*now post any part of the message that is too long with a non-blocking
              send and a different tag */

            hypre_MPI_Isend(post_array[post_array_size], size,
                            hypre_MPI_BYTE, proc, post_tag, 
                            /*hypre_MPI_COMM_WORLD, */
                            comm,
                            &post_send_requests[post_array_size]); 

            post_array_size++;
         }

         /*now append the size information into the overhead storage */
         /* index_ptr =  send_response_buf + max_response_size_bytes; */
         index_ptr = (void *) ((char *) send_response_buf +
                               max_response_size_bytes);

         memcpy(index_ptr, &response_message_size, sizeof(HYPRE_Int));
         
         /*send the block of data that includes the overhead */        
         /* this is a blocking send - the recv has already been posted */
         hypre_MPI_Send(send_response_buf, max_response_total_bytes, 
                        hypre_MPI_BYTE, proc, response_tag, comm); 

         /*--------------------------------------------------------------*/
      
         /* look for any more contact messages*/
         hypre_MPI_Iprobe(hypre_MPI_ANY_SOURCE, contact_tag, comm,
                          &contact_flag, &status);
      }
    
      /* no more contact messages waiting - either 
         (1) check to see if we have received all of our response messages
         (2) participate in termination (check for messages from children)
         (3) participate in termination sweep (check for message from parent) */
    
      if (!responses_complete) 
      {
         hypre_MPI_Testall(num_contacts, response_requests, &responses_complete, 
                           response_statuses);
         if (responses_complete && num_procs == 1) terminate = 1; /*added 11/08 */

      }
      else if(!children_complete) /* have all of our children received all of their 
                                     response messages?*/
      {
         hypre_MPI_Testall(tree.num_child, term_requests, &children_complete, 
                           term_statuses);

         /* if we have gotten term messages from all of our children, send a term 
            message to our parent.  Then post a receive to hear back from parent */ 
	 if (children_complete & (myid > 0)) /*root does not have a parent*/ 
         {
            hypre_MPI_Isend(NULL, 0, HYPRE_MPI_INT, tree.parent_id, term_tag,
                            comm, &request_parent);
            
            hypre_MPI_Irecv(NULL, 0, HYPRE_MPI_INT, tree.parent_id, term_tag,
                            comm, &term_request1);
         }
      }
      else /*have we gotten a term message from our parent? */
      {
         if (myid == 0) /* root doesn't have a parent */
         {
            terminate = 1;
         } 
         else  
         {
            hypre_MPI_Test(&term_request1, &terminate, &term_status1);
         }  
         if (terminate) /*tell children to terminate */
	 {
            if (myid > 0 ) hypre_MPI_Wait(&request_parent, &status_parent);
            
	    for (i=0; i< tree.num_child; i++)
	    {  /*a blocking send  - recv has been posted already*/
	       hypre_MPI_Send(NULL, 0, HYPRE_MPI_INT, tree.child_id[i],
                              term_tag, comm);
	    }
	 }
      } 
   }
Esempio n. 5
0
HYPRE_Int
hypre_CreateCommInfoFromStencil( hypre_StructGrid      *grid,
                                 hypre_StructStencil   *stencil,
                                 hypre_CommInfo       **comm_info_ptr )
{
   HYPRE_Int              i,j,k, d, m, s;   

   hypre_BoxArrayArray   *send_boxes;
   hypre_BoxArrayArray   *recv_boxes;

   HYPRE_Int            **send_procs;
   HYPRE_Int            **recv_procs;
   HYPRE_Int            **send_rboxnums;
   HYPRE_Int            **recv_rboxnums;
   hypre_BoxArrayArray   *send_rboxes;
   hypre_BoxArrayArray   *recv_rboxes;

   hypre_BoxArray        *local_boxes;
   HYPRE_Int              num_boxes;

   HYPRE_Int             *local_ids;

   hypre_BoxManager      *boxman;
                       
   hypre_Index           *stencil_shape;
   hypre_IndexRef         stencil_offset;
   hypre_IndexRef         pshift;
                          
   hypre_Box             *box;
   hypre_Box             *hood_box;
   hypre_Box             *grow_box;
   hypre_Box             *extend_box;
   hypre_Box             *int_box;
   hypre_Box             *periodic_box;
   
   HYPRE_Int              stencil_grid[3][3][3];
   HYPRE_Int              grow[3][2];
                       
   hypre_BoxManEntry    **entries;
   hypre_BoxManEntry     *entry;
   
   HYPRE_Int              num_entries;
   hypre_BoxArray        *neighbor_boxes = NULL;
   HYPRE_Int             *neighbor_procs = NULL;
   HYPRE_Int             *neighbor_ids = NULL;
   HYPRE_Int             *neighbor_shifts = NULL;
   HYPRE_Int              neighbor_count;
   HYPRE_Int              neighbor_alloc;

   hypre_Index            ilower, iupper;

   hypre_BoxArray        *send_box_array;
   hypre_BoxArray        *recv_box_array;
   hypre_BoxArray        *send_rbox_array;
   hypre_BoxArray        *recv_rbox_array;
                       
   hypre_Box            **cboxes;
   hypre_Box             *cboxes_mem;
   HYPRE_Int             *cboxes_neighbor_location;
   HYPRE_Int              num_cboxes, cbox_alloc;
                       
   HYPRE_Int              istart[3], istop[3];
   HYPRE_Int              sgindex[3];               

   HYPRE_Int              num_periods, loc, box_id, id, proc_id;
   HYPRE_Int              myid;
   
   MPI_Comm               comm;

   /*------------------------------------------------------
    * Initializations
    *------------------------------------------------------*/

   local_boxes  = hypre_StructGridBoxes(grid);
   local_ids    = hypre_StructGridIDs(grid);
   num_boxes    = hypre_BoxArraySize(local_boxes);
   num_periods  = hypre_StructGridNumPeriods(grid);
   
   boxman    = hypre_StructGridBoxMan(grid);
   comm      =  hypre_StructGridComm(grid);
   
   hypre_MPI_Comm_rank(comm, &myid);
  
   for (k = 0; k < 3; k++)
   {
      for (j = 0; j < 3; j++)
      {
         for (i = 0; i < 3; i++)
         {
            stencil_grid[i][j][k] = 0;
         }
      }
   }

   /*------------------------------------------------------
    * Compute the "grow" information from the stencil
    *------------------------------------------------------*/

   stencil_shape = hypre_StructStencilShape(stencil);

   for (d = 0; d < 3; d++)
   {
      grow[d][0] = 0;
      grow[d][1] = 0;
   }

   for (s = 0; s < hypre_StructStencilSize(stencil); s++)
   {
      stencil_offset = stencil_shape[s];

      for (d = 0; d < 3; d++)
      {
         m = stencil_offset[d];

         istart[d] = 1;
         istop[d]  = 1;

         if (m < 0)
         {
            istart[d] = 0;
            grow[d][0] = hypre_max(grow[d][0], -m);
         }
         else if (m > 0)
         {
            istop[d] = 2;
            grow[d][1] = hypre_max(grow[d][1],  m);
         }
      }

      /* update stencil grid from the grow_stencil */
      for (k = istart[2]; k <= istop[2]; k++)
      {
         for (j = istart[1]; j <= istop[1]; j++)
         {
            for (i = istart[0]; i <= istop[0]; i++)
            {
               stencil_grid[i][j][k] = 1;
            }
         }
      }
   }

   /*------------------------------------------------------
    * Compute send/recv boxes and procs for each local box
    *------------------------------------------------------*/

   /* initialize: for each local box, we create an array of send/recv info */

   send_boxes = hypre_BoxArrayArrayCreate(num_boxes);
   recv_boxes = hypre_BoxArrayArrayCreate(num_boxes);
   send_procs = hypre_CTAlloc(HYPRE_Int *, num_boxes);
   recv_procs = hypre_CTAlloc(HYPRE_Int *, num_boxes);

   /* Remote boxnums and boxes describe data on the opposing processor, so some
      shifting of boxes is needed below for periodic neighbor boxes.  Remote box
      info is also needed for receives to allow for reverse communication. */
   send_rboxnums = hypre_CTAlloc(HYPRE_Int *, num_boxes);
   send_rboxes   = hypre_BoxArrayArrayCreate(num_boxes);
   recv_rboxnums = hypre_CTAlloc(HYPRE_Int *, num_boxes);
   recv_rboxes   = hypre_BoxArrayArrayCreate(num_boxes);

   grow_box = hypre_BoxCreate();
   extend_box = hypre_BoxCreate();
   int_box  = hypre_BoxCreate();
   periodic_box =  hypre_BoxCreate();
 
   /* storage we will use and keep track of the neighbors */
   neighbor_alloc = 30; /* initial guess at max size */
   neighbor_boxes = hypre_BoxArrayCreate(neighbor_alloc);
   neighbor_procs = hypre_CTAlloc(HYPRE_Int, neighbor_alloc);
   neighbor_ids = hypre_CTAlloc(HYPRE_Int, neighbor_alloc);
   neighbor_shifts = hypre_CTAlloc(HYPRE_Int, neighbor_alloc);

   /* storage we will use to collect all of the intersected boxes (the send and
      recv regions for box i (this may not be enough in the case of periodic
      boxes, so we will have to check) */
   cbox_alloc =  hypre_BoxManNEntries(boxman);

   cboxes_neighbor_location = hypre_CTAlloc(HYPRE_Int, cbox_alloc);
   cboxes = hypre_CTAlloc(hypre_Box *, cbox_alloc);
   cboxes_mem = hypre_CTAlloc(hypre_Box, cbox_alloc);

   /******* loop through each local box **************/

   for (i = 0; i < num_boxes; i++)
   {
      /* get the box */
      box = hypre_BoxArrayBox(local_boxes, i);
      /* box_id = local_ids[i]; the box id in the Box Manager is the box number,
       * and we use this to find out if a box has intersected with itself */
      box_id = i;
      
      /* grow box local i according to the stencil*/
      hypre_CopyBox(box, grow_box);
      for (d = 0; d < 3; d++)
      {
         hypre_BoxIMinD(grow_box, d) -= grow[d][0];
         hypre_BoxIMaxD(grow_box, d) += grow[d][1];
      }

      /* extend_box - to find the list of potential neighbors, we need to grow
         the local box a bit differently in case, for example, the stencil grows
         in one dimension [0] and not the other [1] */
      hypre_CopyBox(box, extend_box);
      for (d = 0; d < 3; d++)
      { 
         hypre_BoxIMinD(extend_box, d) -= hypre_max(grow[d][0],grow[d][1]);
         hypre_BoxIMaxD(extend_box, d) += hypre_max(grow[d][0],grow[d][1]);
      }

      /*------------------------------------------------
       * Determine the neighbors of box i
       *------------------------------------------------*/
     
      /* Do this by intersecting the extend box with the BoxManager. 
         We must also check for periodic neighbors. */

      neighbor_count = 0;
      hypre_BoxArraySetSize(neighbor_boxes, 0);
      /* shift the box by each period (k=0 is original box) */
      for (k = 0; k < num_periods; k++)
      {
         hypre_CopyBox(extend_box, periodic_box);
         pshift = hypre_StructGridPShift(grid, k);
         hypre_BoxShiftPos(periodic_box, pshift);
         
         /* get the intersections */
         hypre_BoxManIntersect(boxman, hypre_BoxIMin(periodic_box) , 
                               hypre_BoxIMax(periodic_box) , 
                               &entries , &num_entries);
      
         /* note: do we need to remove the intersection with our original box?
            no if periodic, yes if non-periodic (k=0) */ 

         /* unpack entries (first check storage) */
         if (neighbor_count + num_entries > neighbor_alloc)
         {
            neighbor_alloc = neighbor_count + num_entries + 5;
            neighbor_procs = hypre_TReAlloc(neighbor_procs, HYPRE_Int,
                                            neighbor_alloc);
            neighbor_ids = hypre_TReAlloc(neighbor_ids, HYPRE_Int, neighbor_alloc);
            neighbor_shifts = hypre_TReAlloc(neighbor_shifts, HYPRE_Int,
                                             neighbor_alloc);
         }
         /* check storage for the array */
         hypre_BoxArraySetSize(neighbor_boxes, neighbor_count + num_entries);
         /* now unpack */
         for (j = 0; j < num_entries; j++)
         {
            entry = entries[j];
            proc_id = hypre_BoxManEntryProc(entry);        
            id = hypre_BoxManEntryId(entry); 
            /* don't keep box i in the non-periodic case*/  
            if (!k)
            {
               if((myid == proc_id) && (box_id == id))
               {
                  continue;
               }
            }

            hypre_BoxManEntryGetExtents(entry, ilower, iupper);        
            hypre_BoxSetExtents(hypre_BoxArrayBox(neighbor_boxes, neighbor_count),
                                ilower, iupper);
            /* shift the periodic boxes (needs to be the opposite of above) */
            if (k)
            {
               hypre_BoxShiftNeg(
                  hypre_BoxArrayBox(neighbor_boxes, neighbor_count), pshift);
            }
            
            neighbor_procs[neighbor_count] = proc_id;
            neighbor_ids[neighbor_count] = id;
            neighbor_shifts[neighbor_count] = k;
            neighbor_count++;
         }
         hypre_BoxArraySetSize(neighbor_boxes, neighbor_count);

         hypre_TFree(entries);
  
      } /* end of loop through periods k */

      /* Now we have a list of all of the neighbors for box i! */

      /* note: we don't want/need to remove duplicates - they should have
         different intersections (TO DO: put more thought into if there are ever
         any exceptions to this? - the intersection routine already eliminates
         duplicates - so what i mean is eliminating duplicates from multiple
         intersection calls in periodic case)  */  
    
      /*------------------------------------------------
       * Compute recv_box_array for box i
       *------------------------------------------------*/

      /* check size of storage for cboxes */
      /* let's make sure that we have enough storage in case each neighbor
         produces a send/recv region */
      if (neighbor_count > cbox_alloc)
      {
         cbox_alloc = neighbor_count;
         cboxes_neighbor_location = hypre_TReAlloc(cboxes_neighbor_location, 
                                                   HYPRE_Int, cbox_alloc);
         cboxes = hypre_TReAlloc(cboxes, hypre_Box *, cbox_alloc);
         cboxes_mem = hypre_TReAlloc(cboxes_mem, hypre_Box, cbox_alloc);
      }

      /* Loop through each neighbor box.  If the neighbor box intersects the
         grown box i (grown according to our stencil), then the intersection is
         a recv region.  If the neighbor box was shifted to handle periodicity,
         we need to (positive) shift it back. */

      num_cboxes = 0;
      
      for (k = 0; k < neighbor_count; k++)
      {
         hood_box = hypre_BoxArrayBox(neighbor_boxes, k);
         /* check the stencil grid to see if it makes sense to intersect */
         for (d = 0; d < 3; d++)
         {
            sgindex[d] = 1;
               
            s = hypre_BoxIMinD(hood_box, d) - hypre_BoxIMaxD(box, d);
            if (s > 0)
            {
               sgindex[d] = 2;
            }
            s = hypre_BoxIMinD(box, d) - hypre_BoxIMaxD(hood_box, d);
            if (s > 0)
            {
               sgindex[d] = 0;
            }
         }
         /* it makes sense only if we have at least one non-zero entry */   
         if (stencil_grid[sgindex[0]][sgindex[1]][sgindex[2]])
         {
            /* intersect - result is int_box */
            hypre_IntersectBoxes(grow_box, hood_box, int_box);
            /* if we have a positive volume box, this is a recv region */
            if (hypre_BoxVolume(int_box))
            {
               /* keep track of which neighbor: k... */
               cboxes_neighbor_location[num_cboxes] = k;
               cboxes[num_cboxes] = &cboxes_mem[num_cboxes];
               /* keep the intersected box */
               hypre_CopyBox(int_box, cboxes[num_cboxes]);
               num_cboxes++;
            }
         }
      } /* end of loop through each neighbor */

      /* create recv_box_array and recv_procs for box i */
      recv_box_array = hypre_BoxArrayArrayBoxArray(recv_boxes, i);
      hypre_BoxArraySetSize(recv_box_array, num_cboxes);
      recv_procs[i] = hypre_CTAlloc(HYPRE_Int, num_cboxes);
      recv_rboxnums[i] = hypre_CTAlloc(HYPRE_Int, num_cboxes);
      recv_rbox_array = hypre_BoxArrayArrayBoxArray(recv_rboxes, i);
      hypre_BoxArraySetSize(recv_rbox_array, num_cboxes);

      for (m = 0; m < num_cboxes; m++)
      {
         loc = cboxes_neighbor_location[m];
         recv_procs[i][m] = neighbor_procs[loc];
         recv_rboxnums[i][m] = neighbor_ids[loc];
         hypre_CopyBox(cboxes[m], hypre_BoxArrayBox(recv_box_array, m));

         /* if periodic, positive shift before copying to the rbox_array */
         if (neighbor_shifts[loc]) /* periodic if shift != 0 */
         {
            pshift = hypre_StructGridPShift(grid, neighbor_shifts[loc]);
            hypre_BoxShiftPos(cboxes[m], pshift);
         }
         hypre_CopyBox(cboxes[m], hypre_BoxArrayBox(recv_rbox_array, m));

         cboxes[m] = NULL;
      }

      /*------------------------------------------------
       * Compute send_box_array for box i
       *------------------------------------------------*/

      /* Loop through each neighbor box.  If the grown neighbor box intersects
         box i, then the intersection is a send region.  If the neighbor box was
         shifted to handle periodicity, we need to (positive) shift it back. */

      num_cboxes = 0;

      for (k = 0; k < neighbor_count; k++)
      {
         hood_box = hypre_BoxArrayBox(neighbor_boxes, k);
         /* check the stencil grid to see if it makes sense to intersect */
         for (d = 0; d < 3; d++)
         {
            sgindex[d] = 1;
            
            s = hypre_BoxIMinD(box, d) - hypre_BoxIMaxD(hood_box, d);
            if (s > 0)
            {
               sgindex[d] = 2;
            }
            s = hypre_BoxIMinD(hood_box, d) - hypre_BoxIMaxD(box, d);
            if (s > 0)
            {
               sgindex[d] = 0;
            }
         }
         /* it makes sense only if we have at least one non-zero entry */   
         if (stencil_grid[sgindex[0]][sgindex[1]][sgindex[2]])
         {
            /* grow the neighbor box and intersect */
            hypre_CopyBox(hood_box, grow_box);
            for (d = 0; d < 3; d++)
            {
               hypre_BoxIMinD(grow_box, d) -= grow[d][0];
               hypre_BoxIMaxD(grow_box, d) += grow[d][1];
            }
            hypre_IntersectBoxes(box, grow_box, int_box);
            /* if we have a positive volume box, this is a send region */
            if (hypre_BoxVolume(int_box))
            {
               /* keep track of which neighbor: k... */
               cboxes_neighbor_location[num_cboxes] = k;
               cboxes[num_cboxes] = &cboxes_mem[num_cboxes];
               /* keep the intersected box */
               hypre_CopyBox(int_box, cboxes[num_cboxes]);
               num_cboxes++;
            }
         }
      }/* end of loop through neighbors */

      /* create send_box_array and send_procs for box i */
      send_box_array = hypre_BoxArrayArrayBoxArray(send_boxes, i);
      hypre_BoxArraySetSize(send_box_array, num_cboxes);
      send_procs[i] = hypre_CTAlloc(HYPRE_Int, num_cboxes);
      send_rboxnums[i] = hypre_CTAlloc(HYPRE_Int, num_cboxes);
      send_rbox_array = hypre_BoxArrayArrayBoxArray(send_rboxes, i);
      hypre_BoxArraySetSize(send_rbox_array, num_cboxes);

      for (m = 0; m < num_cboxes; m++)
      {
         loc = cboxes_neighbor_location[m];
         send_procs[i][m] = neighbor_procs[loc];
         send_rboxnums[i][m] = neighbor_ids[loc];
         hypre_CopyBox(cboxes[m], hypre_BoxArrayBox(send_box_array, m));

         /* if periodic, positive shift before copying to the rbox_array */
         if (neighbor_shifts[loc]) /* periodic if shift != 0 */
         {
            pshift = hypre_StructGridPShift(grid, neighbor_shifts[loc]);
            hypre_BoxShiftPos(cboxes[m], pshift);
         }
         hypre_CopyBox(cboxes[m], hypre_BoxArrayBox(send_rbox_array, m));

         cboxes[m] = NULL;
      }
   } /* end of loop through each local box */