int
hypre_InitializeCommunication( hypre_CommPkg *comm_pkg,
double *send_data,
double *recv_data,
hypre_CommHandle **comm_handle_ptr )
{
int ierr = 0;
hypre_CommHandle *comm_handle;
int num_sends = hypre_CommPkgNumSends(comm_pkg);
int num_recvs = hypre_CommPkgNumRecvs(comm_pkg);
MPI_Comm comm = hypre_CommPkgComm(comm_pkg);
int num_requests;
MPI_Request *requests;
MPI_Status *status;
double **send_buffers;
double **recv_buffers;
int *send_sizes;
int *recv_sizes;
hypre_CommType *send_type;
hypre_CommTypeEntry *send_entry;
hypre_CommType *recv_type;
hypre_CommTypeEntry *recv_entry;
int *length_array;
int *stride_array;
double *iptr, *jptr, *kptr, *lptr, *bptr;
int i, j, k, ii, jj, kk, ll;
int entry_size, total_size;
/*--------------------------------------------------------------------
* allocate requests and status
*--------------------------------------------------------------------*/
num_requests = num_sends + num_recvs;
requests = hypre_CTAlloc(MPI_Request, num_requests);
status = hypre_CTAlloc(MPI_Status, num_requests);
/*--------------------------------------------------------------------
* allocate buffers
*--------------------------------------------------------------------*/
/* allocate send buffers */
send_buffers = hypre_TAlloc(double *, num_sends);
send_sizes = hypre_TAlloc(int, num_sends);
total_size = 0;
for (i = 0; i < num_sends; i++)
{
send_type = hypre_CommPkgSendType(comm_pkg, i);
send_sizes[i] = 0;
for (j = 0; j < hypre_CommTypeNumEntries(send_type); j++)
{
send_entry = hypre_CommTypeCommEntry(send_type, j);
length_array = hypre_CommTypeEntryLengthArray(send_entry);
entry_size = 1;
for (k = 0; k < 4; k++)
{
entry_size *= length_array[k];
}
send_sizes[i] += entry_size;
}
total_size += send_sizes[i];
}
if (num_sends > 0)
{
send_buffers[0] = hypre_SharedTAlloc(double, total_size);
for (i = 1; i < num_sends; i++)
{
send_buffers[i] = send_buffers[i-1] + send_sizes[i-1];
}
}
hypre_CommPkg *
hypre_CommPkgCreate( hypre_BoxArrayArray *send_boxes,
hypre_BoxArrayArray *recv_boxes,
hypre_Index send_stride,
hypre_Index recv_stride,
hypre_BoxArray *send_data_space,
hypre_BoxArray *recv_data_space,
int **send_processes,
int **recv_processes,
int num_values,
MPI_Comm comm,
hypre_Index periodic )
{
hypre_CommPkg *comm_pkg;
int num_sends;
int *send_procs;
hypre_CommType **send_types;
int num_recvs;
int *recv_procs;
hypre_CommType **recv_types;
hypre_CommType *copy_from_type;
hypre_CommType *copy_to_type;
int i;
/*------------------------------------------------------
* Put arguments into hypre_CommPkg
*------------------------------------------------------*/
comm_pkg = hypre_CTAlloc(hypre_CommPkg, 1);
hypre_CommPkgNumValues(comm_pkg) = num_values;
hypre_CommPkgComm(comm_pkg) = comm;
/*------------------------------------------------------
* Set up communication information
*------------------------------------------------------*/
hypre_CommPkgCreateInfo(send_boxes, send_stride,
send_data_space, send_processes,
num_values, comm, periodic,
&num_sends, &send_procs,
&send_types, ©_from_type);
hypre_CommPkgNumSends(comm_pkg) = num_sends;
hypre_CommPkgSendProcs(comm_pkg) = send_procs;
hypre_CommPkgSendTypes(comm_pkg) = send_types;
hypre_CommPkgCopyFromType(comm_pkg) = copy_from_type;
hypre_CommPkgCreateInfo(recv_boxes, recv_stride,
recv_data_space, recv_processes,
num_values, comm, periodic,
&num_recvs, &recv_procs,
&recv_types, ©_to_type);
hypre_CommPkgNumRecvs(comm_pkg) = num_recvs;
hypre_CommPkgRecvProcs(comm_pkg) = recv_procs;
hypre_CommPkgRecvTypes(comm_pkg) = recv_types;
hypre_CommPkgCopyToType(comm_pkg) = copy_to_type;
/*------------------------------------------------------
* Destroy the input boxes and processes
*------------------------------------------------------*/
hypre_ForBoxArrayI(i, send_boxes)
hypre_TFree(send_processes[i]);
hypre_BoxArrayArrayDestroy(send_boxes);
hypre_TFree(send_processes);
hypre_ForBoxArrayI(i, recv_boxes)
hypre_TFree(recv_processes[i]);
hypre_BoxArrayArrayDestroy(recv_boxes);
hypre_TFree(recv_processes);
#if defined(HYPRE_COMM_SIMPLE) || defined(HYPRE_COMM_VOLATILE)
#else
hypre_CommPkgCommit(comm_pkg);
/* free up comm types */
for (i = 0; i < hypre_CommPkgNumSends(comm_pkg); i++)
hypre_CommTypeDestroy(hypre_CommPkgSendType(comm_pkg, i));
hypre_TFree(hypre_CommPkgSendTypes(comm_pkg));
for (i = 0; i < hypre_CommPkgNumRecvs(comm_pkg); i++)
hypre_CommTypeDestroy(hypre_CommPkgRecvType(comm_pkg, i));
hypre_TFree(hypre_CommPkgRecvTypes(comm_pkg));
#endif
return comm_pkg;
}
int
hypre_CommPkgCreate( hypre_CommInfo *comm_info,
hypre_BoxArray *send_data_space,
hypre_BoxArray *recv_data_space,
int num_values,
MPI_Comm comm,
hypre_CommPkg **comm_pkg_ptr )
{
int ierr = 0;
hypre_BoxArrayArray *send_boxes;
hypre_BoxArrayArray *recv_boxes;
hypre_IndexRef send_stride;
hypre_IndexRef recv_stride;
int **send_processes;
int **recv_processes;
int **send_rboxnums;
hypre_BoxArrayArray *send_rboxes;
hypre_CommPkg *comm_pkg;
hypre_CommType *comm_types;
hypre_CommType *comm_type;
hypre_CommEntryType *ct_entries;
int *ct_loc_boxnums;
int *ct_rem_boxnums;
hypre_Box *ct_loc_boxes;
hypre_Box *ct_rem_boxes;
int *p_comm_types;
int num_comms, num_entries, comm_bufsize;
hypre_BoxArray *box_array;
hypre_Box *box;
hypre_BoxArray *rbox_array;
hypre_Box *rbox;
hypre_Box *data_box;
int *data_offsets;
int data_offset;
int i, j, k, p, m, size;
int num_procs, my_proc;
/*------------------------------------------------------
*------------------------------------------------------*/
send_boxes = hypre_CommInfoSendBoxes(comm_info);
recv_boxes = hypre_CommInfoRecvBoxes(comm_info);
send_stride = hypre_CommInfoSendStride(comm_info);
recv_stride = hypre_CommInfoRecvStride(comm_info);
send_processes = hypre_CommInfoSendProcesses(comm_info);
recv_processes = hypre_CommInfoRecvProcesses(comm_info);
send_rboxnums = hypre_CommInfoSendRBoxnums(comm_info);
send_rboxes = hypre_CommInfoSendRBoxes(comm_info);
MPI_Comm_size(comm, &num_procs );
MPI_Comm_rank(comm, &my_proc );
/*------------------------------------------------------
* Set up various entries in CommPkg
*------------------------------------------------------*/
comm_pkg = hypre_CTAlloc(hypre_CommPkg, 1);
hypre_CommPkgComm(comm_pkg) = comm;
hypre_CommPkgFirstSend(comm_pkg) = 1;
hypre_CommPkgFirstRecv(comm_pkg) = 1;
hypre_CommPkgNumValues(comm_pkg) = num_values;
hypre_CopyIndex(send_stride, hypre_CommPkgSendStride(comm_pkg));
hypre_CopyIndex(recv_stride, hypre_CommPkgRecvStride(comm_pkg));
/*------------------------------------------------------
* Set up send CommType information
*------------------------------------------------------*/
p_comm_types = hypre_CTAlloc(int, num_procs);
/* set send_data_offsets and send_data_space */
data_offsets = hypre_TAlloc(int, hypre_BoxArraySize(send_data_space));
data_offset = 0;
hypre_ForBoxI(i, send_data_space)
{
data_offsets[i] = data_offset;
data_box = hypre_BoxArrayBox(send_data_space, i);
data_offset += hypre_BoxVolume(data_box) * num_values;
}