static int search_complete(uint64_t tag, size_t msglen, MPID_Request * rreq) { int mpi_errno = MPI_SUCCESS; BEGIN_FUNC(FCNAME); rreq->status.MPI_SOURCE = get_source(tag); rreq->status.MPI_TAG = get_tag(tag); rreq->status.MPI_ERROR = MPI_SUCCESS; MPIR_STATUS_SET_COUNT(rreq->status, msglen); END_FUNC(FCNAME); return mpi_errno; }
int ADD_SUFFIX(MPID_nem_ofi_anysource_iprobe)(int tag, MPIR_Comm * comm, int context_offset, int *flag, MPI_Status * status) { int rc; BEGIN_FUNC(FCNAME); *flag = NORMAL_PEEK; rc = ADD_SUFFIX(MPID_nem_ofi_iprobe)(NULL, MPI_ANY_SOURCE, tag, comm, context_offset, flag, status); END_FUNC(FCNAME); return rc; }
int ADD_SUFFIX(MPID_nem_ofi_iprobe)(struct MPIDI_VC *vc, int source, int tag, MPIR_Comm * comm, int context_offset, int *flag, MPI_Status * status) { int rc; BEGIN_FUNC(FCNAME); *flag = 0; rc = ADD_SUFFIX(MPID_nem_ofi_iprobe_impl)(vc, source, tag, comm, context_offset, flag, status, NULL); END_FUNC(FCNAME); return rc; }
static int ADD_SUFFIX(peek_callback)(cq_tagged_entry_t * wc, MPIR_Request * rreq) { int mpi_errno = MPI_SUCCESS; BEGIN_FUNC(FCNAME); REQ_OFI(rreq)->match_state = PEEK_FOUND; #if API_SET == API_SET_1 rreq->status.MPI_SOURCE = get_source(wc->tag); #elif API_SET == API_SET_2 rreq->status.MPI_SOURCE = wc->data; #endif rreq->status.MPI_TAG = get_tag(wc->tag); MPIR_STATUS_SET_COUNT(rreq->status, wc->len); rreq->status.MPI_ERROR = MPI_SUCCESS; END_FUNC(FCNAME); return mpi_errno; }
static int tsearch_callback(cq_tagged_entry_t * wc, MPID_Request * rreq) { int mpi_errno = MPI_SUCCESS; BEGIN_FUNC(FCNAME); if (wc->data) { REQ_OFI(rreq)->match_state = TSEARCH_FOUND; rreq->status.MPI_SOURCE = get_source(wc->tag); rreq->status.MPI_TAG = get_tag(wc->tag); MPIR_STATUS_SET_COUNT(rreq->status, wc->len); rreq->status.MPI_ERROR = MPI_SUCCESS; } else { REQ_OFI(rreq)->match_state = TSEARCH_NOT_FOUND; } END_FUNC(FCNAME); return mpi_errno; }
int ADD_SUFFIX(MPID_nem_ofi_improbe)(struct MPIDI_VC *vc, int source, int tag, MPIR_Comm * comm, int context_offset, int *flag, MPIR_Request ** message, MPI_Status * status) { int old_error = status->MPI_ERROR; int s; BEGIN_FUNC(FCNAME); *flag = CLAIM_PEEK; s = ADD_SUFFIX(MPID_nem_ofi_iprobe_impl)(vc, source, tag, comm, context_offset, flag, status, message); if (*flag) { status->MPI_ERROR = old_error; (*message)->kind = MPIR_REQUEST_KIND__MPROBE; } END_FUNC(FCNAME); return s; }
int MPID_nem_ofi_iprobe_impl(struct MPIDI_VC *vc, int source, int tag, MPID_Comm * comm, int context_offset, int *flag, MPI_Status * status, MPID_Request ** rreq_ptr) { int ret, mpi_errno = MPI_SUCCESS; fi_addr_t remote_proc = 0; uint64_t match_bits, mask_bits; size_t len; MPID_Request rreq_s, *rreq; BEGIN_FUNC(FCNAME); if (rreq_ptr) { MPIDI_Request_create_rreq(rreq, mpi_errno, goto fn_exit); *rreq_ptr = rreq; rreq->comm = comm; rreq->dev.match.parts.rank = source; rreq->dev.match.parts.tag = tag; rreq->dev.match.parts.context_id = comm->context_id; MPIR_Comm_add_ref(comm); } else {
int ADD_SUFFIX(MPID_nem_ofi_iprobe_impl)(struct MPIDI_VC *vc, int source, int tag, MPIR_Comm * comm, int context_offset, int *flag, MPI_Status * status, MPIR_Request ** rreq_ptr) { int ret, mpi_errno = MPI_SUCCESS; fi_addr_t remote_proc = 0; uint64_t match_bits, mask_bits; size_t len; MPIR_Request rreq_s, *rreq; BEGIN_FUNC(FCNAME); if (rreq_ptr) { MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_create_req(&rreq, 1)); rreq->kind = MPIR_REQUEST_KIND__RECV; *rreq_ptr = rreq; rreq->comm = comm; rreq->dev.match.parts.rank = source; rreq->dev.match.parts.tag = tag; rreq->dev.match.parts.context_id = comm->context_id; MPIR_Comm_add_ref(comm); } else { rreq = &rreq_s; rreq->dev.OnDataAvail = NULL; } REQ_OFI(rreq)->pack_buffer = NULL; REQ_OFI(rreq)->event_callback = ADD_SUFFIX(peek_callback); REQ_OFI(rreq)->match_state = PEEK_INIT; OFI_ADDR_INIT(source, vc, remote_proc); #if API_SET == API_SET_1 match_bits = init_recvtag(&mask_bits, comm->context_id + context_offset, source, tag); #elif API_SET == API_SET_2 match_bits = init_recvtag_2(&mask_bits, comm->context_id + context_offset, tag); #endif /* ------------------------------------------------------------------------- */ /* fi_recvmsg with FI_PEEK: */ /* Initiate a search for a match in the hardware or software queue. */ /* The search can complete immediately with -ENOMSG. */ /* I successful, libfabric will enqueue a context entry into the completion */ /* queue to make the search nonblocking. This code will poll until the */ /* entry is enqueued. */ /* ------------------------------------------------------------------------- */ msg_tagged_t msg; uint64_t msgflags = FI_PEEK; msg.msg_iov = NULL; msg.desc = NULL; msg.iov_count = 0; msg.addr = remote_proc; msg.tag = match_bits; msg.ignore = mask_bits; msg.context = (void *) &(REQ_OFI(rreq)->ofi_context); msg.data = 0; if(*flag == CLAIM_PEEK) msgflags|=FI_CLAIM; ret = fi_trecvmsg(gl_data.endpoint,&msg,msgflags); if(ret == -ENOMSG) { if (rreq_ptr) { MPIR_Request_free(rreq); *rreq_ptr = NULL; *flag = 0; } MPID_nem_ofi_poll(MPID_NONBLOCKING_POLL); goto fn_exit; } MPIR_ERR_CHKANDJUMP4((ret < 0), mpi_errno, MPI_ERR_OTHER, "**ofi_peek", "**ofi_peek %s %d %s %s", __SHORT_FILE__, __LINE__, FCNAME, fi_strerror(-ret)); while (PEEK_INIT == REQ_OFI(rreq)->match_state) MPID_nem_ofi_poll(MPID_BLOCKING_POLL); if (PEEK_NOT_FOUND == REQ_OFI(rreq)->match_state) { if (rreq_ptr) { MPIR_Request_free(rreq); *rreq_ptr = NULL; *flag = 0; } MPID_nem_ofi_poll(MPID_NONBLOCKING_POLL); goto fn_exit; } if (status != MPI_STATUS_IGNORE) *status = rreq->status; MPIR_Request_add_ref(rreq); *flag = 1; END_FUNC_RC(FCNAME); }
static inline int ADD_SUFFIX(MPID_nem_ofi_recv_callback)(cq_tagged_entry_t * wc, MPID_Request * rreq) { int err0, err1, src, mpi_errno = MPI_SUCCESS; uint64_t ssend_bits; MPIDI_msg_sz_t sz; MPIDI_VC_t *vc; MPID_Request *sync_req; BEGIN_FUNC(FCNAME); /* ---------------------------------------------------- */ /* Populate the MPI Status and unpack noncontig buffer */ /* ---------------------------------------------------- */ rreq->status.MPI_ERROR = MPI_SUCCESS; #if API_SET == API_SET_1 rreq->status.MPI_SOURCE = get_source(wc->tag); #elif API_SET == API_SET_2 rreq->status.MPI_SOURCE = wc->data; #endif src = rreq->status.MPI_SOURCE; rreq->status.MPI_TAG = get_tag(wc->tag); REQ_OFI(rreq)->req_started = 1; MPIR_STATUS_SET_COUNT(rreq->status, wc->len); if (REQ_OFI(rreq)->pack_buffer) { MPIDI_CH3U_Buffer_copy(REQ_OFI(rreq)->pack_buffer, MPIR_STATUS_GET_COUNT(rreq->status), MPI_BYTE, &err0, rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, &sz, &err1); MPIR_STATUS_SET_COUNT(rreq->status, sz); MPIU_Free(REQ_OFI(rreq)->pack_buffer); if (err0 || err1) { rreq->status.MPI_ERROR = MPI_ERR_TYPE; } } if ((wc->tag & MPID_PROTOCOL_MASK) == MPID_SYNC_SEND) { /* ---------------------------------------------------- */ /* Ack the sync send and wait for the send request */ /* completion(when callback executed. A protocol bit */ /* MPID_SYNC_SEND_ACK is set in the tag bits to provide */ /* separation of MPI messages and protocol messages */ /* ---------------------------------------------------- */ vc = REQ_OFI(rreq)->vc; if (!vc) { /* MPI_ANY_SOURCE -- Post message from status, complete the VC */ vc = rreq->comm->dev.vcrt->vcr_table[src]; MPIU_Assert(vc); } #if API_SET == API_SET_1 ssend_bits = init_sendtag(rreq->dev.match.parts.context_id, rreq->comm->rank, rreq->status.MPI_TAG, MPID_SYNC_SEND_ACK); #elif API_SET == API_SET_2 ssend_bits = init_sendtag_2(rreq->dev.match.parts.context_id, rreq->status.MPI_TAG, MPID_SYNC_SEND_ACK); #endif MPID_nem_ofi_create_req(&sync_req, 1); sync_req->dev.OnDataAvail = NULL; sync_req->dev.next = NULL; REQ_OFI(sync_req)->event_callback = MPID_nem_ofi_sync_recv_callback; REQ_OFI(sync_req)->parent = rreq; #if API_SET == API_SET_1 FI_RC_RETRY(fi_tsend(gl_data.endpoint, #elif API_SET == API_SET_2 FI_RC_RETRY(fi_tsenddata(gl_data.endpoint, #endif NULL, 0, gl_data.mr, #if API_SET == API_SET_2 rreq->comm->rank, #endif VC_OFI(vc)->direct_addr, ssend_bits, &(REQ_OFI(sync_req)->ofi_context)), tsend); }
int MPID_nem_ofi_poll(int in_blocking_poll) { int complete = 0, mpi_errno = MPI_SUCCESS; ssize_t ret; cq_tagged_entry_t wc; cq_err_entry_t error; MPIDI_VC_t *vc; MPID_Request *req; req_fn reqFn; BEGIN_FUNC(FCNAME); do { /* ----------------------------------------------------- */ /* Poll the completion queue */ /* The strategy here is */ /* ret>0 successfull poll, events returned */ /* ret==0 empty poll, no events/no error */ /* ret<0, error, but some error instances should not */ /* cause MPI to terminate */ /* ----------------------------------------------------- */ ret = fi_cq_read(gl_data.cq, /* Tagged completion queue */ (void *) &wc, /* OUT: Tagged completion entry */ 1); /* Number of entries to poll */ if (ret > 0) { if (NULL != wc.op_context) { req = context_to_req(wc.op_context); if (REQ_OFI(req)->event_callback) { MPIDI_CH3I_NM_OFI_RC(REQ_OFI(req)->event_callback(&wc, req)); continue; } reqFn = req->dev.OnDataAvail; if (reqFn) { if (REQ_OFI(req)->pack_buffer) { MPIU_Free(REQ_OFI(req)->pack_buffer); } vc = REQ_OFI(req)->vc; complete = 0; MPIDI_CH3I_NM_OFI_RC(reqFn(vc, req, &complete)); continue; } else { MPIU_Assert(0); } } else { MPIU_Assert(0); } } else if (ret == -FI_EAGAIN) ; else if (ret < 0) { if (ret == -FI_EAVAIL) { ret = fi_cq_readerr(gl_data.cq, (void *) &error, 0); if (error.err == FI_ETRUNC) { /* ----------------------------------------------------- */ /* This error message should only be delivered on send */ /* events. We want to ignore truncation errors */ /* on the sender side, but complete the request anyway */ /* Other kinds of requests, this is fatal. */ /* ----------------------------------------------------- */ req = context_to_req(error.op_context); if (req->kind == MPID_REQUEST_SEND) { mpi_errno = REQ_OFI(req)->event_callback(NULL, req); } else if (req->kind == MPID_REQUEST_RECV) { mpi_errno = REQ_OFI(req)->event_callback(&wc, req); req->status.MPI_ERROR = MPI_ERR_TRUNCATE; req->status.MPI_TAG = error.tag; } else { mpi_errno = MPI_ERR_OTHER; } } else if (error.err == FI_ECANCELED) { req = context_to_req(error.op_context); MPIR_STATUS_SET_CANCEL_BIT(req->status, TRUE); } else { mpi_errno = MPI_ERR_OTHER; } } else { MPIR_ERR_CHKANDJUMP4(1, mpi_errno, MPI_ERR_OTHER, "**ofi_poll", "**ofi_poll %s %d %s %s", __SHORT_FILE__, __LINE__, FCNAME, fi_strerror(-ret)); } } } while (in_blocking_poll && (ret > 0)); END_FUNC_RC(FCNAME); }