int MPID_nem_newmad_process_rdtype(MPID_Request **rreq_p, MPID_Datatype * dt_ptr, MPIDI_msg_sz_t data_sz, struct iovec *newmad_iov[], int *num_iov)
{
MPID_Request *rreq = *rreq_p;
MPIDI_msg_sz_t last;
MPID_IOV *iov;
int n_iov = 0;
int mpi_errno = MPI_SUCCESS;
int index;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWMAD_PROCESS_RDTYPE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWMAD_PROCESS_RDTYPE);
if (rreq->dev.segment_ptr == NULL)
{
rreq->dev.segment_ptr = MPID_Segment_alloc( );
MPIU_ERR_CHKANDJUMP1((rreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
}
MPID_Segment_init(rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, rreq->dev.segment_ptr, 0);
rreq->dev.segment_first = 0;
rreq->dev.segment_size = data_sz;
last = rreq->dev.segment_size;
MPID_Segment_count_contig_blocks(rreq->dev.segment_ptr,rreq->dev.segment_first,&last,&n_iov);
MPIU_Assert(n_iov > 0);
iov = MPIU_Malloc(n_iov*sizeof(MPID_IOV));
MPID_Segment_unpack_vector(rreq->dev.segment_ptr, rreq->dev.segment_first, &last,iov, &n_iov);
MPIU_Assert(last == rreq->dev.segment_size);
#ifdef DEBUG
for(index = 0; index < n_iov ; index++)
{
fprintf(stdout,"======================\n");
fprintf(stdout,"RECV iov[%i]: [base %p][len %i]\n",index,
iov[index].MPID_IOV_BUF,iov[index].MPID_IOV_LEN);
}
#endif
if(n_iov <= NMAD_IOV_MAX_DEPTH)
{
for(index=0; index < n_iov ; index++)
{
(*newmad_iov)[index].iov_base = iov[index].MPID_IOV_BUF;
(*newmad_iov)[index].iov_len = iov[index].MPID_IOV_LEN;
}
rreq->dev.tmpbuf = NULL;
*num_iov = n_iov;
}
else
{
int packsize = 0;
MPIR_Pack_size_impl(rreq->dev.user_count, rreq->dev.datatype, &packsize);
rreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
MPIU_Assert(rreq->dev.tmpbuf);
rreq->dev.tmpbuf_sz = packsize;
(*newmad_iov)[0].iov_base = (char *) rreq->dev.tmpbuf;
(*newmad_iov)[0].iov_len = (uint32_t) packsize;
*num_iov = 1 ;
}
MPIU_Free(iov);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_NEWMAD_PROCESS_RDTYPE);
return mpi_errno;
fn_fail: ATTRIBUTE((unused))
goto fn_exit;
}
/* create a request */
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIU_Assert(sreq != NULL);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
MPID_Datatype_get_ptr(datatype, sreq->dev.datatype_ptr);
MPID_Datatype_add_ref(sreq->dev.datatype_ptr);
}
sreq->partner_request = NULL;
sreq->dev.OnDataAvail = NULL;
sreq->dev.tmpbuf = NULL;
sreq->ch.vc = vc;
sreq->ch.noncontig = FALSE;
_dbg_mxm_output(5,
"isSend ========> Sending USER msg for req %p (context %d to %d tag %d size %d) \n",
sreq, comm->context_id + context_offset, rank, tag, data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area-> ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[0].ptr = NULL;
req_area->iov_buf[0].length = 0;
if (data_sz) {
if (dt_contig) {
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb;
req_area->iov_buf[0].length = data_sz;
}
else {
MPIDI_msg_sz_t last;
MPI_Aint packsize = 0;
sreq->ch.noncontig = TRUE;
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER,
"**nomem", "**nomem %s", "MPID_Segment_alloc");
MPIR_Pack_size_impl(count, datatype, &packsize);
last = data_sz;
if (packsize > 0) {
sreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
MPIU_Assert(sreq->dev.tmpbuf);
MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf);
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = sreq->dev.tmpbuf;
req_area->iov_buf[0].length = last;
}
}
}
vc_area->pending_sends += 1;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_SYNC,
(mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag,
comm->context_id
+
context_offset),
0);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
_dbg_mxm_out_req(sreq);
fn_exit:
*sreq_ptr = sreq;
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISSEND);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int _mxm_handle_sreq(MPID_Request * req)
{
int complete = FALSE;
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_dbg_mxm_out_buf(req_area->iov_buf[0].ptr,
(req_area->iov_buf[0].length >
16 ? 16 : req_area->iov_buf[0].length));
vc_area->pending_sends -= 1;
if (((req->dev.datatype_ptr != NULL) && (req->dev.tmpbuf != NULL))) {
MPIU_Free(req->dev.tmpbuf);
}
if (req_area->iov_count > MXM_MPICH_MAX_IOV) {
MPIU_Free(req_area->iov_buf);
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
}
reqFn = req->dev.OnDataAvail;
if (!reqFn) {
MPIDI_CH3U_Request_complete(req);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
MPIDI_VC_t *vc = req->ch.vc;
reqFn(vc, req, &complete);
if (!complete) {
MPIU_Assert(complete == TRUE);
}
}
return complete;
}
static void _mxm_send_completion_cb(void *context)
{
MPID_Request *req = (MPID_Request *) context;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIU_Assert(req);
_dbg_mxm_out_req(req);
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status);
list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue);
_dbg_mxm_output(5, "========> %s SEND req %p status %d\n",
(MPIR_STATUS_GET_CANCEL_BIT(req->status) ? "Canceling" : "Completing"),
req, req->status.MPI_ERROR);
if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) {
_mxm_handle_sreq(req);
}
}
static int _mxm_isend(MPID_nem_mxm_ep_t * ep, MPID_nem_mxm_req_area * req,
int type, mxm_mq_h mxm_mq, int mxm_rank, int id, mxm_tag_t mxm_tag, int block)
{
int mpi_errno = MPI_SUCCESS;
mxm_error_t ret = MXM_OK;
mxm_send_req_t *mxm_sreq;
list_head_t *free_queue = NULL;
MPIU_Assert(ep);
MPIU_Assert(req);
free_queue = &ep->free_queue;
req->mxm_req = list_dequeue_mxm_req(free_queue);
if (!req->mxm_req) {
list_grow_mxm_req(free_queue);
req->mxm_req = list_dequeue_mxm_req(free_queue);
if (!req->mxm_req) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "empty free queue");
mpi_errno = MPI_ERR_OTHER;
goto fn_fail;
}
}
mxm_sreq = &(req->mxm_req->item.send);
mxm_sreq->base.state = MXM_REQ_NEW;
mxm_sreq->base.mq = mxm_mq;
mxm_sreq->base.conn = ep->mxm_conn;
mxm_sreq->base.completed_cb = _mxm_send_completion_cb;
mxm_sreq->base.context = req->ctx;
if (type == MXM_MPICH_ISEND_AM) {
mxm_sreq->opcode = MXM_REQ_OP_AM;
mxm_sreq->flags = 0;
mxm_sreq->op.am.hid = id;
mxm_sreq->op.am.imm_data = mxm_rank;
}
else if (type == MXM_MPICH_ISEND_SYNC) {
mxm_sreq->opcode = MXM_REQ_OP_SEND_SYNC;
mxm_sreq->flags = 0;
mxm_sreq->op.send.tag = mxm_tag;
mxm_sreq->op.send.imm_data = mxm_rank;
}
else {
mxm_sreq->opcode = MXM_REQ_OP_SEND;
mxm_sreq->flags = 0;
mxm_sreq->op.send.tag = mxm_tag;
mxm_sreq->op.send.imm_data = mxm_rank;
}
if (likely(req->iov_count == 1)) {
mxm_sreq->base.data_type = MXM_REQ_DATA_BUFFER;
mxm_sreq->base.data.buffer.ptr = req->iov_buf[0].ptr;
mxm_sreq->base.data.buffer.length = req->iov_buf[0].length;
}
else {
mxm_sreq->base.data_type = MXM_REQ_DATA_IOV;
mxm_sreq->base.data.iov.vector = req->iov_buf;
mxm_sreq->base.data.iov.count = req->iov_count;
}
ret = mxm_req_send(mxm_sreq);
if (MXM_OK != ret) {
list_enqueue(free_queue, &req->mxm_req->queue);
mpi_errno = MPI_ERR_OTHER;
goto fn_fail;
}
if (block)
_mxm_req_wait(&mxm_sreq->base);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
#if 0 /* Consider using this function in case non contiguous data */
static int _mxm_process_sdtype(MPID_Request ** sreq_p, MPI_Datatype datatype,
MPID_Datatype * dt_ptr, MPIDI_msg_sz_t data_sz, const void *buf,
int count, mxm_req_buffer_t ** iov_buf, int *iov_count)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq = *sreq_p;
MPIDI_msg_sz_t last;
MPID_IOV *iov;
int n_iov = 0;
int index;
int size_to_copy = 0;
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem",
"**nomem %s", "MPID_Segment_alloc");
MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = data_sz;
last = sreq->dev.segment_size;
MPID_Segment_count_contig_blocks(sreq->dev.segment_ptr, sreq->dev.segment_first, &last,
(MPI_Aint *) & n_iov);
MPIU_Assert(n_iov > 0);
iov = MPIU_Malloc(n_iov * sizeof(*iov));
MPIU_Assert(iov);
last = sreq->dev.segment_size;
MPID_Segment_pack_vector(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, iov, &n_iov);
MPIU_Assert(last == sreq->dev.segment_size);
#if defined(MXM_DEBUG) && (MXM_DEBUG > 0)
_dbg_mxm_output(7, "Send Noncontiguous data vector %i entries (free slots : %i)\n", n_iov, MXM_REQ_DATA_MAX_IOV);
for(index = 0; index < n_iov; index++) {
_dbg_mxm_output(7, "======= Recv iov[%i] = ptr : %p, len : %i \n",
index, iov[index].MPID_IOV_BUF, iov[index].MPID_IOV_LEN);
}
#endif
if (n_iov > MXM_MPICH_MAX_IOV) {
*iov_buf = (mxm_req_buffer_t *) MPIU_Malloc(n_iov * sizeof(**iov_buf));
MPIU_Assert(*iov_buf);
}
for (index = 0; index < n_iov; index++) {
if (index < (MXM_REQ_DATA_MAX_IOV - 1)) {
(*iov_buf)[index].ptr = iov[index].MPID_IOV_BUF;
(*iov_buf)[index].length = iov[index].MPID_IOV_LEN;
}
else {
size_to_copy += iov[index].MPID_IOV_LEN;
}
}
if (size_to_copy == 0) {
sreq->dev.tmpbuf = NULL;
sreq->dev.tmpbuf_sz = 0;
*iov_count = n_iov;
}
else {
int offset = 0;
sreq->dev.tmpbuf = MPIU_Malloc(size_to_copy);
sreq->dev.tmpbuf_sz = size_to_copy;
MPIU_Assert(sreq->dev.tmpbuf);
for (index = (MXM_REQ_DATA_MAX_IOV - 1); index < n_iov; index++) {
MPIU_Memcpy((char *) (sreq->dev.tmpbuf) + offset, iov[index].MPID_IOV_BUF,
iov[index].MPID_IOV_LEN);
offset += iov[index].MPID_IOV_LEN;
}
(*iov_buf)[MXM_REQ_DATA_MAX_IOV - 1].ptr = sreq->dev.tmpbuf;
(*iov_buf)[MXM_REQ_DATA_MAX_IOV - 1].length = size_to_copy;
*iov_count = MXM_REQ_DATA_MAX_IOV;
}
MPIU_Free(iov);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int _mxm_process_rdtype(MPID_Request ** rreq_p, MPI_Datatype datatype,
MPID_Datatype * dt_ptr, MPIDI_msg_sz_t data_sz, const void *buf,
int count, mxm_req_buffer_t ** iov_buf, int *iov_count)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *rreq = *rreq_p;
MPIDI_msg_sz_t last;
MPL_IOV *iov;
int n_iov = 0;
int index;
if (rreq->dev.segment_ptr == NULL) {
rreq->dev.segment_ptr = MPID_Segment_alloc();
MPIR_ERR_CHKANDJUMP1((rreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem",
"**nomem %s", "MPID_Segment_alloc");
}
MPID_Segment_init(buf, count, datatype, rreq->dev.segment_ptr, 0);
rreq->dev.segment_first = 0;
rreq->dev.segment_size = data_sz;
last = rreq->dev.segment_size;
MPID_Segment_count_contig_blocks(rreq->dev.segment_ptr, rreq->dev.segment_first, &last,
(MPI_Aint *) & n_iov);
MPIU_Assert(n_iov > 0);
iov = MPIU_Malloc(n_iov * sizeof(*iov));
MPIU_Assert(iov);
last = rreq->dev.segment_size;
MPID_Segment_unpack_vector(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, iov, &n_iov);
MPIU_Assert(last == rreq->dev.segment_size);
#if defined(MXM_DEBUG) && (MXM_DEBUG > 0)
_dbg_mxm_output(7, "Recv Noncontiguous data vector %i entries (free slots : %i)\n", n_iov,
MXM_REQ_DATA_MAX_IOV);
for (index = 0; index < n_iov; index++) {
_dbg_mxm_output(7, "======= Recv iov[%i] = ptr : %p, len : %i \n",
index, iov[index].MPL_IOV_BUF, iov[index].MPL_IOV_LEN);
}
#endif
if (n_iov <= MXM_REQ_DATA_MAX_IOV) {
if (n_iov > MXM_MPICH_MAX_IOV) {
*iov_buf = (mxm_req_buffer_t *) MPIU_Malloc(n_iov * sizeof(**iov_buf));
MPIU_Assert(*iov_buf);
}
for (index = 0; index < n_iov; index++) {
(*iov_buf)[index].ptr = iov[index].MPL_IOV_BUF;
(*iov_buf)[index].length = iov[index].MPL_IOV_LEN;
}
rreq->dev.tmpbuf = NULL;
rreq->dev.tmpbuf_sz = 0;
*iov_count = n_iov;
}
else {
MPI_Aint packsize = 0;
MPIR_Pack_size_impl(rreq->dev.user_count, rreq->dev.datatype, &packsize);
rreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
MPIU_Assert(rreq->dev.tmpbuf);
rreq->dev.tmpbuf_sz = packsize;
(*iov_buf)[0].ptr = rreq->dev.tmpbuf;
(*iov_buf)[0].length = (size_t) packsize;
*iov_count = 1;
}
MPIU_Free(iov);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
