static inline __opal_attribute_always_inline__ int ompi_mtl_mxm_recv_init(mca_mtl_mxm_request_t *mtl_mxm_request, opal_convertor_t *convertor, mxm_recv_req_t *mxm_recv_req) { int ret; mtl_mxm_request->convertor = convertor; ret = ompi_mtl_datatype_recv_buf(convertor, &mtl_mxm_request->buf, &mtl_mxm_request->length, &mtl_mxm_request->free_after); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } mxm_recv_req->base.state = MXM_REQ_NEW; mxm_recv_req->base.flags = 0; mxm_recv_req->base.data_type = MXM_REQ_DATA_BUFFER; mxm_recv_req->base.data.buffer.ptr = mtl_mxm_request->buf; mxm_recv_req->base.data.buffer.length = mtl_mxm_request->length; mxm_recv_req->base.data.buffer.mkey = MXM_MKEY_NONE; mxm_recv_req->base.context = mtl_mxm_request; mxm_recv_req->base.completed_cb = ompi_mtl_mxm_recv_completion_cb; return OMPI_SUCCESS; }
int ompi_mtl_mxm_irecv(struct mca_mtl_base_module_t* mtl, struct ompi_communicator_t *comm, int src, int tag, struct opal_convertor_t *convertor, struct mca_mtl_request_t *mtl_request) { mca_mtl_mxm_request_t * mtl_mxm_request; mxm_error_t err; mxm_recv_req_t *mxm_recv_req; int ret; mtl_mxm_request = (mca_mtl_mxm_request_t*) mtl_request; mtl_mxm_request->convertor = convertor; ret = ompi_mtl_datatype_recv_buf(mtl_mxm_request->convertor, &mtl_mxm_request->buf, &mtl_mxm_request->length, &mtl_mxm_request->free_after); if (OMPI_SUCCESS != ret) { return ret; } /* prepare a receive request embedded in the MTL request */ mxm_recv_req = &mtl_mxm_request->mxm.recv; #if MXM_API >= MXM_VERSION(2,0) mtl_mxm_request->is_send = 0; #endif mxm_recv_req->base.state = MXM_REQ_NEW; ompi_mtl_mxm_set_recv_envelope(mxm_recv_req, comm, src, tag); #if MXM_API < MXM_VERSION(2,0) mxm_recv_req->base.flags = 0; #endif mxm_recv_req->base.data_type = MXM_REQ_DATA_BUFFER; mxm_recv_req->base.data.buffer.ptr = mtl_mxm_request->buf; mxm_recv_req->base.data.buffer.length = mtl_mxm_request->length; #if MXM_API < MXM_VERSION(1,5) mxm_recv_req->base.data.buffer.mkey = MXM_MKEY_NONE; #else mxm_recv_req->base.data.buffer.memh = MXM_INVALID_MEM_HANDLE; #endif mxm_recv_req->base.context = mtl_mxm_request; mxm_recv_req->base.completed_cb = ompi_mtl_mxm_recv_completion_cb; /* post-recv */ err = mxm_req_recv(mxm_recv_req); if (MXM_OK != err) { orte_show_help("help-mtl-mxm.txt", "error posting receive", true, mxm_error_string(err), mtl_mxm_request->buf, mtl_mxm_request->length); return OMPI_ERROR; } return OMPI_SUCCESS; }
int ompi_mtl_psm_irecv(struct mca_mtl_base_module_t* mtl, struct ompi_communicator_t *comm, int src, int tag, struct opal_convertor_t *convertor, struct mca_mtl_request_t *mtl_request) { int ret; psm_error_t err; mca_mtl_psm_request_t * mtl_psm_request = (mca_mtl_psm_request_t*) mtl_request; uint64_t mqtag; uint64_t tagsel; size_t length; ret = ompi_mtl_datatype_recv_buf(convertor, &mtl_psm_request->buf, &length, &mtl_psm_request->free_after); if (OMPI_SUCCESS != ret) return ret; mtl_psm_request->length = length; mtl_psm_request->convertor = convertor; mtl_psm_request->type = OMPI_MTL_PSM_IRECV; PSM_MAKE_TAGSEL(src, tag, comm->c_contextid, mqtag, tagsel); #if 0 printf("recv bits: 0x%016llx 0x%016llx\n", mqtag, tagsel); #endif err = psm_mq_irecv(ompi_mtl_psm.mq, mqtag, tagsel, 0, mtl_psm_request->buf, length, mtl_psm_request, &mtl_psm_request->psm_request); if (err) { orte_show_help("help-mtl-psm.txt", "error posting receive", true, psm_error_get_string(err), mtl_psm_request->buf, length); return OMPI_ERROR; } return OMPI_SUCCESS; }
int ompi_mtl_mxm_irecv(struct mca_mtl_base_module_t* mtl, struct ompi_communicator_t *comm, int src, int tag, struct opal_convertor_t *convertor, struct mca_mtl_request_t *mtl_request) { mca_mtl_mxm_request_t * mtl_mxm_request; mxm_error_t err; mxm_recv_req_t *mxm_recv_req; int ret; mtl_mxm_request = (mca_mtl_mxm_request_t*) mtl_request; mtl_mxm_request->convertor = convertor; ret = ompi_mtl_datatype_recv_buf(mtl_mxm_request->convertor, &mtl_mxm_request->buf, &mtl_mxm_request->length, &mtl_mxm_request->free_after); if (OMPI_SUCCESS != ret) { return ret; } /* prepare a receive request embedded in the MTL request */ mxm_recv_req = &mtl_mxm_request->mxm.recv; mxm_recv_req->base.state = MXM_REQ_NEW; mxm_recv_req->base.mq = ompi_mtl_mxm_mq_lookup(comm); mxm_recv_req->base.conn = (src == MPI_ANY_SOURCE) ? NULL : ompi_mtl_mxm_conn_lookup(comm, src); mxm_recv_req->base.flags = 0; mxm_recv_req->base.data_type = MXM_REQ_DATA_BUFFER; mxm_recv_req->base.data.buffer.ptr = mtl_mxm_request->buf; mxm_recv_req->base.data.buffer.length = mtl_mxm_request->length; mxm_recv_req->base.data.buffer.mkey = MXM_MKEY_NONE; mxm_recv_req->base.context = mtl_mxm_request; mxm_recv_req->base.completed_cb = ompi_mtl_mxm_recv_completion_cb; mxm_recv_req->tag = tag; mxm_recv_req->tag_mask = (tag == MPI_ANY_TAG) ? 0 : 0xffffffffU; /* post-recv */ err = mxm_req_recv(mxm_recv_req); if (MXM_OK != err) { orte_show_help("help-mtl-mxm.txt", "error posting receive", true, mxm_error_string(err), mtl_mxm_request->buf, mtl_mxm_request->length); return OMPI_ERROR; } return OMPI_SUCCESS; }
int ompi_mtl_portals4_imrecv(struct mca_mtl_base_module_t* mtl, struct opal_convertor_t *convertor, struct ompi_message_t **message, struct mca_mtl_request_t *mtl_request) { ompi_mtl_portals4_recv_request_t *ptl_request = (ompi_mtl_portals4_recv_request_t*) mtl_request; void *start; size_t length; bool free_after; int ret; ompi_mtl_portals4_message_t *ptl_message = (ompi_mtl_portals4_message_t*) (*message)->req_ptr; ret = ompi_mtl_datatype_recv_buf(convertor, &start, &length, &free_after); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } #if OPAL_ENABLE_DEBUG ptl_request->opcount = OPAL_THREAD_ADD64((int64_t*) &ompi_mtl_portals4.recv_opcount, 1); ptl_request->hdr_data = 0; #endif ptl_request->super.type = portals4_req_recv; ptl_request->super.event_callback = ompi_mtl_portals4_recv_progress; ptl_request->buffer_ptr = (free_after) ? start : NULL; ptl_request->convertor = convertor; ptl_request->delivery_ptr = start; ptl_request->delivery_len = length; ptl_request->super.super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS; ptl_request->pending_reply = 0; OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output, "Mrecv %lu of length %ld (0x%lx)\n", ptl_request->opcount, (int64_t)length, (unsigned long) ptl_request)); (*message) = MPI_MESSAGE_NULL; return ompi_mtl_portals4_recv_progress(&(ptl_message->ev), &ptl_request->super); }
int ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl, struct ompi_communicator_t *comm, int src, int tag, struct opal_convertor_t *convertor, mca_mtl_request_t *mtl_request) { ptl_match_bits_t match_bits, ignore_bits; int ret = OMPI_SUCCESS; ptl_process_t remote_proc; ompi_mtl_portals4_recv_request_t *ptl_request = (ompi_mtl_portals4_recv_request_t*) mtl_request; void *start; size_t length; bool free_after; ptl_me_t me; if (MPI_ANY_SOURCE == src) { if (ompi_mtl_portals4.use_logical) { remote_proc.rank = PTL_RANK_ANY; } else { remote_proc.phys.nid = PTL_NID_ANY; remote_proc.phys.pid = PTL_PID_ANY; } } else if ((ompi_mtl_portals4.use_logical) && (MPI_COMM_WORLD == comm)) { remote_proc.rank = src; } else { ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, src ); remote_proc = *((ptl_process_t*) ompi_mtl_portals4_get_endpoint (mtl, ompi_proc)); } MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid, src, tag); ret = ompi_mtl_datatype_recv_buf(convertor, &start, &length, &free_after); if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { return ret; } ptl_request->super.type = portals4_req_recv; ptl_request->super.event_callback = ompi_mtl_portals4_recv_progress; #if OPAL_ENABLE_DEBUG ptl_request->opcount = OPAL_THREAD_ADD64((int64_t*) &ompi_mtl_portals4.recv_opcount, 1); ptl_request->hdr_data = 0; #endif ptl_request->buffer_ptr = (free_after) ? start : NULL; ptl_request->convertor = convertor; ptl_request->delivery_ptr = start; ptl_request->delivery_len = length; ptl_request->req_started = false; ptl_request->super.super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS; ptl_request->pending_reply = 0; OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_framework.framework_output, "Recv %lu from %x,%x of length %ld (0x%lx, 0x%lx, 0x%lx)\n", ptl_request->opcount, remote_proc.phys.nid, remote_proc.phys.pid, (int64_t)length, match_bits, ignore_bits, (unsigned long) ptl_request)); me.start = start; me.length = length; me.ct_handle = PTL_CT_NONE; me.min_free = 0; me.uid = ompi_mtl_portals4.uid; me.options = PTL_ME_OP_PUT | PTL_ME_USE_ONCE | PTL_ME_EVENT_UNLINK_DISABLE; if (length <= ompi_mtl_portals4.short_limit) { me.options |= PTL_ME_EVENT_LINK_DISABLE; } me.match_id = remote_proc; me.match_bits = match_bits; me.ignore_bits = ignore_bits; ret = PtlMEAppend(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.recv_idx, &me, PTL_PRIORITY_LIST, ptl_request, &ptl_request->me_h); if (OPAL_UNLIKELY(PTL_OK != ret)) { if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr); opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: PtlMEAppend failed: %d", __FILE__, __LINE__, ret); return ompi_mtl_portals4_get_error(ret); } /* if a long message, spin until we either have a comm event or a link event, guaranteeing progress for long unexpected messages. */ if (length > ompi_mtl_portals4.short_limit) { while (true != ptl_request->req_started) { ompi_mtl_portals4_progress(); } } return OMPI_SUCCESS; }
static int ompi_mtl_portals_get_data(ompi_mtl_portals_event_t *recv_event, struct ompi_convertor_t *convertor, ompi_mtl_portals_request_t *ptl_request) { int ret; ptl_md_t md; ptl_handle_md_t md_h; size_t buflen; if (PTL_IS_SHORT_MSG(recv_event->ev.match_bits)) { /* the buffer is sitting in the short message queue */ struct iovec iov; uint32_t iov_count = 1; size_t max_data; ompi_mtl_portals_recv_short_block_t *block = recv_event->ev.md.user_ptr; iov.iov_base = (((char*) recv_event->ev.md.start) + recv_event->ev.offset); iov.iov_len = recv_event->ev.mlength; max_data = iov.iov_len; /* see if this message filled the receive block */ if (recv_event->ev.md.length - (recv_event->ev.offset + recv_event->ev.mlength) < recv_event->ev.md.max_size) { block->full = true; } /* pull out the data */ if (iov.iov_len > 0) { ompi_convertor_unpack(convertor, &iov, &iov_count, &max_data ); } /* if synchronous, return an ack */ if (PTL_IS_SYNC_MSG(recv_event->ev)) { md.length = 0; md.start = (((char*) recv_event->ev.md.start) + recv_event->ev.offset); md.threshold = 1; /* send */ md.options = PTL_MD_EVENT_START_DISABLE; md.user_ptr = NULL; md.eq_handle = ompi_mtl_portals.ptl_eq_h; ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h, md, PTL_UNLINK, &md_h); if (PTL_OK != ret) { opal_output(fileno(stderr)," Error returned from PtlMDBind. Error code - %d \n",ret); abort(); } OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "acking recv: 0x%016llx\n", recv_event->ev.match_bits)); ret = PtlPut(md_h, PTL_NO_ACK_REQ, recv_event->ev.initiator, OMPI_MTL_PORTALS_ACK_TABLE_ID, 0, recv_event->ev.hdr_data, 0, 0); if (PTL_OK != ret) { opal_output(fileno(stderr)," Error returned from PtlPut. Error code - %d \n",ret); abort(); } } /* finished with our buffer space */ ompi_mtl_portals_return_block_part(&ompi_mtl_portals, block); ompi_convertor_get_packed_size(convertor, &buflen); ptl_request->super.ompi_req->req_status.MPI_SOURCE = PTL_GET_SOURCE(recv_event->ev.match_bits); ptl_request->super.ompi_req->req_status.MPI_TAG = PTL_GET_TAG(recv_event->ev.match_bits); ptl_request->super.ompi_req->req_status.MPI_ERROR = (recv_event->ev.rlength > buflen) ? MPI_ERR_TRUNCATE : MPI_SUCCESS; ptl_request->super.ompi_req->req_status._count = recv_event->ev.mlength; OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "recv complete: 0x%016llx\n", recv_event->ev.match_bits)); ptl_request->super.completion_callback(&ptl_request->super); } else { ret = ompi_mtl_datatype_recv_buf(convertor, &md.start, &buflen, &ptl_request->free_after); if (OMPI_SUCCESS != ret) { opal_output(fileno(stderr)," Error returned from ompi_mtl_datatype_recv_buf. Error code - %d \n",ret); abort(); } md.length = (recv_event->ev.rlength > buflen) ? buflen : recv_event->ev.rlength; md.threshold = 2; /* send and get */ md.options = PTL_MD_EVENT_START_DISABLE; md.user_ptr = ptl_request; md.eq_handle = ompi_mtl_portals.ptl_eq_h; /* retain because it's unclear how many events we'll get here. Some implementations give just the REPLY, others give SEND and REPLY */ ret = PtlMDBind(ompi_mtl_portals.ptl_ni_h, md, PTL_RETAIN, &md_h); if (PTL_OK != ret) { opal_output(fileno(stderr)," Error returned from ompi_mtl_datatype_recv_buf. Error code - %d \n",ret); abort(); } ptl_request->event_callback = ompi_mtl_portals_recv_progress; ret = PtlGet(md_h, recv_event->ev.initiator, OMPI_MTL_PORTALS_READ_TABLE_ID, 0, recv_event->ev.hdr_data, 0); if (PTL_OK != ret) { opal_output(fileno(stderr)," Error returned from PtlGet. Error code - %d \n",ret); abort(); } ptl_request->super.ompi_req->req_status.MPI_SOURCE = PTL_GET_SOURCE(recv_event->ev.match_bits); ptl_request->super.ompi_req->req_status.MPI_TAG = PTL_GET_TAG(recv_event->ev.match_bits); ptl_request->super.ompi_req->req_status.MPI_ERROR = (recv_event->ev.rlength > buflen) ? MPI_ERR_TRUNCATE : MPI_SUCCESS; } return OMPI_SUCCESS; }
int ompi_mtl_portals_irecv(struct mca_mtl_base_module_t* mtl, struct ompi_communicator_t *comm, int src, int tag, struct ompi_convertor_t *convertor, mca_mtl_request_t *mtl_request) { ptl_match_bits_t match_bits, ignore_bits; ptl_md_t md; ptl_handle_md_t md_h; ptl_handle_me_t me_h; int ret; ptl_process_id_t remote_proc; mca_mtl_base_endpoint_t *endpoint = NULL; ompi_mtl_portals_request_t *ptl_request = (ompi_mtl_portals_request_t*) mtl_request; ompi_mtl_portals_event_t *recv_event = NULL; size_t buflen; ptl_request->convertor = convertor; if (MPI_ANY_SOURCE == src) { remote_proc.nid = PTL_NID_ANY; remote_proc.pid = PTL_PID_ANY; } else { ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, src ); endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml; remote_proc = endpoint->ptl_proc; } PTL_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid, src, tag); OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "recv bits: 0x%016llx 0x%016llx\n", match_bits, ignore_bits)); /* first, check the queue of processed unexpected messages */ recv_event = ompi_mtl_portals_search_unex_q(match_bits, ignore_bits); if (NULL != recv_event) { /* found it */ ompi_mtl_portals_get_data(recv_event, convertor, ptl_request); OMPI_FREE_LIST_RETURN(&ompi_mtl_portals.event_fl, (ompi_free_list_item_t*)recv_event); goto cleanup; } else { restart_search: /* check unexpected events */ recv_event = ompi_mtl_portals_search_unex_events(match_bits, ignore_bits); if (NULL != recv_event) { /* found it */ ompi_mtl_portals_get_data(recv_event, convertor, ptl_request); OMPI_FREE_LIST_RETURN(&ompi_mtl_portals.event_fl, (ompi_free_list_item_t*)recv_event); goto cleanup; } } /* didn't find it, now post the receive */ ret = ompi_mtl_datatype_recv_buf(convertor, &md.start, &buflen, &ptl_request->free_after); md.length = buflen; /* create ME entry */ ret = PtlMEInsert(ompi_mtl_portals.ptl_match_ins_me_h, remote_proc, match_bits, ignore_bits, PTL_UNLINK, PTL_INS_BEFORE, &me_h); if( ret !=PTL_OK) { return ompi_common_portals_error_ptl_to_ompi(ret); } /* associate a memory descriptor with the Match list Entry */ md.threshold = 0; md.options = PTL_MD_OP_PUT | PTL_MD_TRUNCATE | PTL_MD_EVENT_START_DISABLE; md.user_ptr = ptl_request; md.eq_handle = ompi_mtl_portals.ptl_eq_h; ret=PtlMDAttach(me_h, md, PTL_UNLINK, &md_h); if( ret !=PTL_OK) { return ompi_common_portals_error_ptl_to_ompi(ret); } /* now try to make active */ md.threshold = 1; /* enable the memory descritor, if the ptl_unexpected_recv_eq_h * queue is empty */ ret = PtlMDUpdate(md_h, NULL, &md, ompi_mtl_portals.ptl_unexpected_recv_eq_h); if (ret == PTL_MD_NO_UPDATE) { /* a message has arrived since we searched - look again */ PtlMDUnlink(md_h); if (ptl_request->free_after) { free(md.start); } goto restart_search; } else if( PTL_OK != ret ) { return ompi_common_portals_error_ptl_to_ompi(ret); } ptl_request->event_callback = ompi_mtl_portals_recv_progress; cleanup: return OMPI_SUCCESS; }
int ompi_mtl_portals4_irecv(struct mca_mtl_base_module_t* mtl, struct ompi_communicator_t *comm, int src, int tag, struct opal_convertor_t *convertor, mca_mtl_request_t *mtl_request) { ptl_match_bits_t match_bits, ignore_bits; int ret = OMPI_SUCCESS; ptl_process_t remote_proc; mca_mtl_base_endpoint_t *endpoint = NULL; ompi_mtl_portals4_recv_request_t *ptl_request = (ompi_mtl_portals4_recv_request_t*) mtl_request; void *start; size_t length; bool free_after; ptl_me_t me; if (MPI_ANY_SOURCE == src) { remote_proc.phys.nid = PTL_NID_ANY; remote_proc.phys.pid = PTL_PID_ANY; } else { ompi_proc_t* ompi_proc = ompi_comm_peer_lookup( comm, src ); endpoint = (mca_mtl_base_endpoint_t*) ompi_proc->proc_pml; remote_proc = endpoint->ptl_proc; } MTL_PORTALS4_SET_RECV_BITS(match_bits, ignore_bits, comm->c_contextid, src, tag); ret = ompi_mtl_datatype_recv_buf(convertor, &start, &length, &free_after); if (OMPI_SUCCESS != ret) { opal_output_verbose(1, ompi_mtl_base_output, "%s:%d: PtlMEAppend failed: %d", __FILE__, __LINE__, ret); return ret; } #if OPAL_ENABLE_DEBUG ptl_request->opcount = ++ompi_mtl_portals4.recv_opcount; ptl_request->hdr_data = 0; #endif ptl_request->super.event_callback = ompi_mtl_portals4_recv_progress; ptl_request->buffer_ptr = (free_after) ? start : NULL; ptl_request->convertor = convertor; ptl_request->delivery_ptr = start; ptl_request->delivery_len = length; ptl_request->super.super.ompi_req->req_status.MPI_ERROR = OMPI_SUCCESS; OPAL_OUTPUT_VERBOSE((50, ompi_mtl_base_output, "Recv %d from %x,%x of length %d (0x%lx, 0x%lx)\n", ptl_request->opcount, remote_proc.phys.nid, remote_proc.phys.pid, (int)length, match_bits, ignore_bits)); me.start = start; me.length = length; me.ct_handle = PTL_CT_NONE; me.min_free = 0; me.uid = PTL_UID_ANY; me.options = PTL_ME_OP_PUT | PTL_ME_USE_ONCE | PTL_ME_EVENT_UNLINK_DISABLE; me.match_id = remote_proc; me.match_bits = match_bits; me.ignore_bits = ignore_bits; ret = PtlMEAppend(ompi_mtl_portals4.ni_h, ompi_mtl_portals4.send_idx, &me, PTL_PRIORITY_LIST, ptl_request, &ptl_request->me_h); if (PTL_OK != ret) { if (NULL != ptl_request->buffer_ptr) free(ptl_request->buffer_ptr); opal_output_verbose(1, ompi_mtl_base_output, "%s:%d: PtlMEAppend failed: %d", __FILE__, __LINE__, ret); return ompi_mtl_portals4_get_error(ret); } return OMPI_SUCCESS; }