hypre_ParCSRCommMultiHandle *
hypre_ParCSRCommMultiHandleCreate (HYPRE_Int job,
hypre_ParCSRCommPkg *comm_pkg,
void *send_data,
void *recv_data,
HYPRE_Int num_vecs )
{
HYPRE_Int num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
HYPRE_Int num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg);
MPI_Comm comm = hypre_ParCSRCommPkgComm(comm_pkg);
hypre_ParCSRCommMultiHandle *comm_handle;
HYPRE_Int num_requests;
hypre_MPI_Request *requests;
HYPRE_Int i, j;
HYPRE_Int my_id, num_procs;
HYPRE_Int ip, vec_start, vec_len;
/*--------------------------------------------------------------------
* hypre_Initialize sets up a communication handle,
* posts receives and initiates sends. It always requires num_sends,
* num_recvs, recv_procs and send_procs to be set in comm_pkg.
* There are different options for job:
* job = 1 : is used to initialize communication exchange for the parts
* of vector needed to perform a Matvec, it requires send_data
* and recv_data to be doubles, recv_vec_starts and
* send_map_starts need to be set in comm_pkg.
* job = 2 : is used to initialize communication exchange for the parts
* of vector needed to perform a MatvecT, it requires send_data
* and recv_data to be doubles, recv_vec_starts and
* send_map_starts need to be set in comm_pkg.
*--------------------------------------------------------------------*/
num_requests = num_sends + num_recvs;
requests = hypre_CTAlloc(hypre_MPI_Request, num_requests);
hypre_MPI_Comm_size(comm,&num_procs);
hypre_MPI_Comm_rank(comm,&my_id);
j = 0;
switch (job)
{
case 1:
{
double *d_send_data = (double *) send_data;
double *d_recv_data = (double *) recv_data;
for (i = 0; i < num_recvs; i++)
{
ip = hypre_ParCSRCommPkgRecvProc(comm_pkg, i);
vec_start = hypre_ParCSRCommPkgRecvVecStart(comm_pkg,i);
vec_len = hypre_ParCSRCommPkgRecvVecStart(comm_pkg,i+1)-vec_start;
hypre_MPI_Irecv(&d_recv_data[vec_start*num_vecs], vec_len*num_vecs,
hypre_MPI_DOUBLE, ip, 0, comm, &requests[j++]);
}
for (i = 0; i < num_sends; i++)
{
vec_start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
vec_len = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1)-vec_start;
ip = hypre_ParCSRCommPkgSendProc(comm_pkg, i);
hypre_MPI_Isend(&d_send_data[vec_start*num_vecs], vec_len*num_vecs,
hypre_MPI_DOUBLE, ip, 0, comm, &requests[j++]);
}
break;
}
case 2:
{
double *d_send_data = (double *) send_data;
double *d_recv_data = (double *) recv_data;
for (i = 0; i < num_sends; i++)
{
vec_start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
vec_len = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1) - vec_start;
ip = hypre_ParCSRCommPkgSendProc(comm_pkg, i);
hypre_MPI_Irecv(&d_recv_data[vec_start*num_vecs], vec_len*num_vecs,
hypre_MPI_DOUBLE, ip, 0, comm, &requests[j++]);
}
for (i = 0; i < num_recvs; i++)
{
ip = hypre_ParCSRCommPkgRecvProc(comm_pkg, i);
vec_start = hypre_ParCSRCommPkgRecvVecStart(comm_pkg,i);
vec_len = hypre_ParCSRCommPkgRecvVecStart(comm_pkg,i+1)-vec_start;
hypre_MPI_Isend(&d_send_data[vec_start*num_vecs], vec_len*num_vecs,
hypre_MPI_DOUBLE, ip, 0, comm, &requests[j++]);
}
break;
}
}
/*--------------------------------------------------------------------
* set up comm_handle and return
*--------------------------------------------------------------------*/
comm_handle = hypre_CTAlloc(hypre_ParCSRCommMultiHandle, 1);
hypre_ParCSRCommMultiHandleCommPkg(comm_handle) = comm_pkg;
hypre_ParCSRCommMultiHandleSendData(comm_handle) = send_data;
hypre_ParCSRCommMultiHandleRecvData(comm_handle) = recv_data;
hypre_ParCSRCommMultiHandleNumRequests(comm_handle) = num_requests;
hypre_ParCSRCommMultiHandleRequests(comm_handle) = requests;
return (comm_handle);
}
/* Based on hypre_ParCSRCommHandleCreate in hypre's par_csr_communication.c. The
input variable job controls the communication type: 1=Matvec, 2=MatvecT. */
hypre_ParCSRCommHandle *
hypre_ParCSRCommHandleCreate_bool(HYPRE_Int job,
hypre_ParCSRCommPkg *comm_pkg,
HYPRE_Bool *send_data,
HYPRE_Bool *recv_data)
{
HYPRE_Int num_sends = hypre_ParCSRCommPkgNumSends(comm_pkg);
HYPRE_Int num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg);
MPI_Comm comm = hypre_ParCSRCommPkgComm(comm_pkg);
hypre_ParCSRCommHandle *comm_handle;
HYPRE_Int num_requests;
hypre_MPI_Request *requests;
HYPRE_Int i, j;
HYPRE_Int my_id, num_procs;
HYPRE_Int ip, vec_start, vec_len;
num_requests = num_sends + num_recvs;
requests = hypre_CTAlloc(hypre_MPI_Request, num_requests);
hypre_MPI_Comm_size(comm, &num_procs);
hypre_MPI_Comm_rank(comm, &my_id);
j = 0;
switch (job)
{
case 1:
{
HYPRE_Bool *d_send_data = (HYPRE_Bool *) send_data;
HYPRE_Bool *d_recv_data = (HYPRE_Bool *) recv_data;
for (i = 0; i < num_recvs; i++)
{
ip = hypre_ParCSRCommPkgRecvProc(comm_pkg, i);
vec_start = hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i);
vec_len = hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i+1)-vec_start;
hypre_MPI_Irecv(&d_recv_data[vec_start], vec_len, HYPRE_MPI_BOOL,
ip, 0, comm, &requests[j++]);
}
for (i = 0; i < num_sends; i++)
{
vec_start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
vec_len = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1)-vec_start;
ip = hypre_ParCSRCommPkgSendProc(comm_pkg, i);
hypre_MPI_Isend(&d_send_data[vec_start], vec_len, HYPRE_MPI_BOOL,
ip, 0, comm, &requests[j++]);
}
break;
}
case 2:
{
HYPRE_Bool *d_send_data = (HYPRE_Bool *) send_data;
HYPRE_Bool *d_recv_data = (HYPRE_Bool *) recv_data;
for (i = 0; i < num_sends; i++)
{
vec_start = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i);
vec_len = hypre_ParCSRCommPkgSendMapStart(comm_pkg, i+1)-vec_start;
ip = hypre_ParCSRCommPkgSendProc(comm_pkg, i);
hypre_MPI_Irecv(&d_recv_data[vec_start], vec_len, HYPRE_MPI_BOOL,
ip, 0, comm, &requests[j++]);
}
for (i = 0; i < num_recvs; i++)
{
ip = hypre_ParCSRCommPkgRecvProc(comm_pkg, i);
vec_start = hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i);
vec_len = hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i+1)-vec_start;
hypre_MPI_Isend(&d_send_data[vec_start], vec_len, HYPRE_MPI_BOOL,
ip, 0, comm, &requests[j++]);
}
break;
}
}
/*--------------------------------------------------------------------
* set up comm_handle and return
*--------------------------------------------------------------------*/
comm_handle = hypre_CTAlloc(hypre_ParCSRCommHandle, 1);
hypre_ParCSRCommHandleCommPkg(comm_handle) = comm_pkg;
hypre_ParCSRCommHandleSendData(comm_handle) = send_data;
hypre_ParCSRCommHandleRecvData(comm_handle) = recv_data;
hypre_ParCSRCommHandleNumRequests(comm_handle) = num_requests;
hypre_ParCSRCommHandleRequests(comm_handle) = requests;
return comm_handle;
}
