static ucs_status_t progress_local_cq(uct_ugni_smsg_iface_t *iface){ gni_return_t ugni_rc; gni_cq_entry_t event_data; uct_ugni_smsg_desc_t message_data; uct_ugni_smsg_desc_t *message_pointer; ugni_rc = GNI_CqGetEvent(iface->super.local_cq, &event_data); if(GNI_RC_NOT_DONE == ugni_rc){ return UCS_OK; } if((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)){ /* TODO: handle overruns */ ucs_error("Error posting data. CQ overrun = %d", (int)GNI_CQ_OVERRUN(event_data)); return UCS_ERR_NO_RESOURCE; } message_data.msg_id = GNI_CQ_GET_MSG_ID(event_data); message_pointer = sglib_hashed_uct_ugni_smsg_desc_t_find_member(iface->smsg_list,&message_data); ucs_assert(NULL != message_pointer); message_pointer->ep->outstanding--; iface->super.outstanding--; uct_ugni_ep_check_flush(message_pointer->ep); sglib_hashed_uct_ugni_smsg_desc_t_delete(iface->smsg_list,message_pointer); ucs_mpool_put(message_pointer); return UCS_INPROGRESS; }
static void progress_local_cq(uct_ugni_smsg_iface_t *iface){ gni_return_t ugni_rc; gni_cq_entry_t event_data; uct_ugni_smsg_desc_t message_data; uct_ugni_smsg_desc_t *message_pointer; if(0 == iface->super.outstanding){ return; } ugni_rc = GNI_CqGetEvent(iface->super.local_cq, &event_data); if(GNI_RC_NOT_DONE == ugni_rc){ return; } if((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)){ /* TODO: handle overruns */ ucs_error("Error posting data. CQ overrun = %d", (int)GNI_CQ_OVERRUN(event_data)); return; } message_data.msg_id = GNI_CQ_GET_MSG_ID(event_data); message_pointer = sglib_hashed_uct_ugni_smsg_desc_t_find_member(iface->smsg_list,&message_data); message_pointer->ep->outstanding--; iface->super.outstanding--; sglib_hashed_uct_ugni_smsg_desc_t_delete(iface->smsg_list,message_pointer); ucs_mpool_put(message_pointer); }
ucs_status_t progress_remote_cq(uct_ugni_smsg_iface_t *iface) { gni_return_t ugni_rc; gni_cq_entry_t event_data; uct_ugni_ep_t tl_ep; uct_ugni_ep_t *ugni_ep; uct_ugni_smsg_ep_t *ep; ugni_rc = GNI_CqGetEvent(iface->remote_cq, &event_data); if(GNI_RC_NOT_DONE == ugni_rc){ return UCS_OK; } if (GNI_RC_SUCCESS != ugni_rc || !GNI_CQ_STATUS_OK(event_data) || GNI_CQ_OVERRUN(event_data)) { if(GNI_RC_ERROR_RESOURCE == ugni_rc || (GNI_RC_SUCCESS == ugni_rc && GNI_CQ_OVERRUN(event_data))){ ucs_debug("Detected remote CQ overrun. ungi_rc = %d [%s]", ugni_rc, gni_err_str[ugni_rc]); uct_ugni_smsg_handle_remote_overflow(iface); return UCS_OK; } ucs_error("GNI_CqGetEvent falied with unhandled error. Error status %s %d ", gni_err_str[ugni_rc], ugni_rc); return UCS_ERR_IO_ERROR; } tl_ep.hash_key = GNI_CQ_GET_INST_ID(event_data); ugni_ep = sglib_hashed_uct_ugni_ep_t_find_member(iface->super.eps, &tl_ep); ep = ucs_derived_of(ugni_ep, uct_ugni_smsg_ep_t); process_mbox(iface, ep); return UCS_INPROGRESS; }
static inline int mca_btl_ugni_handle_remote_smsg_overrun (mca_btl_ugni_module_t *btl) { gni_cq_entry_t event_data; unsigned int ep_index; int count, rc; BTL_VERBOSE(("btl/ugni_component detected SMSG CQ overrun. " "processing message backlog...")); /* we don't know which endpoint lost an smsg completion. clear the smsg remote cq and check all mailboxes */ /* clear out remote cq */ do { rc = GNI_CqGetEvent (btl->smsg_remote_cq, &event_data); } while (GNI_RC_NOT_DONE != rc); for (ep_index = 0, count = 0 ; ep_index < btl->endpoint_count ; ++ep_index) { mca_btl_base_endpoint_t *ep = btl->endpoints[ep_index]; if (NULL == ep || MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state) { continue; } /* clear out smsg mailbox */ rc = mca_btl_ugni_smsg_process (ep); if (OPAL_LIKELY(rc >= 0)) { count += rc; } } return count; }
static void __nic_get_completed_txd(struct gnix_nic *nic, gni_cq_handle_t hw_cq, struct gnix_tx_descriptor **txd, gni_return_t *tx_status) { gni_post_descriptor_t *gni_desc; struct gnix_tx_descriptor *txd_p = NULL; struct gnix_fab_req *req; gni_return_t status; int msg_id; gni_cq_entry_t cqe; uint32_t recov = 1; if (__gnix_nic_txd_err_get(nic, &txd_p)) { *txd = txd_p; *tx_status = GNI_RC_TRANSACTION_ERROR; return; } status = GNI_CqGetEvent(hw_cq, &cqe); if (status == GNI_RC_NOT_DONE) { *txd = NULL; *tx_status = GNI_RC_NOT_DONE; return; } assert(status == GNI_RC_SUCCESS || status == GNI_RC_TRANSACTION_ERROR); if (unlikely(status == GNI_RC_TRANSACTION_ERROR)) { status = GNI_CqErrorRecoverable(cqe, &recov); if (status == GNI_RC_SUCCESS) { if (!recov) { char ebuf[512]; GNI_CqErrorStr(cqe, ebuf, sizeof(ebuf)); GNIX_WARN(FI_LOG_EP_DATA, "CQ error status: %s\n", ebuf); } } else { GNIX_WARN(FI_LOG_EP_DATA, "GNI_CqErrorRecover returned: %s\n", gni_err_str[status]); recov = 0; /* assume something bad has happened */ } } if (GNI_CQ_GET_TYPE(cqe) == GNI_CQ_EVENT_TYPE_POST) { status = GNI_GetCompleted(hw_cq, cqe, &gni_desc); assert(status == GNI_RC_SUCCESS || status == GNI_RC_TRANSACTION_ERROR); txd_p = container_of(gni_desc, struct gnix_tx_descriptor, gni_desc); } else if (GNI_CQ_GET_TYPE(cqe) == GNI_CQ_EVENT_TYPE_SMSG) {
static inline int mca_btl_ugni_progress_rdma (mca_btl_ugni_module_t *ugni_module) { ompi_common_ugni_post_desc_t *desc; mca_btl_ugni_base_frag_t *frag; gni_cq_entry_t event_data = 0; uint32_t recoverable = 1; gni_return_t rc; rc = GNI_CqGetEvent (ugni_module->rdma_local_cq, &event_data); if (GNI_RC_NOT_DONE == rc) { return 0; } if (OPAL_UNLIKELY((GNI_RC_SUCCESS != rc && !event_data) || GNI_CQ_OVERRUN(event_data))) { /* TODO -- need to handle overrun -- how do we do this without an event? will the event eventually come back? Ask Cray */ BTL_ERROR(("unhandled post error! ugni rc = %d", rc)); assert (0); return ompi_common_rc_ugni_to_ompi (rc); } rc = GNI_GetCompleted (ugni_module->rdma_local_cq, event_data, (gni_post_descriptor_t **) &desc); if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc && GNI_RC_TRANSACTION_ERROR != rc)) { BTL_ERROR(("Error in GNI_GetComplete %s", gni_err_str[rc])); return ompi_common_rc_ugni_to_ompi (rc); } frag = MCA_BTL_UGNI_DESC_TO_FRAG(desc); if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc || !GNI_CQ_STATUS_OK(event_data))) { (void) GNI_CqErrorRecoverable (event_data, &recoverable); if (OPAL_UNLIKELY(++frag->post_desc.tries >= mca_btl_ugni_component.rdma_max_retries || !recoverable)) { /* give up */ BTL_ERROR(("giving up on frag %p", (void *) frag)); frag->cbfunc (frag, OMPI_ERROR); return OMPI_ERROR; } /* repost transaction */ mca_btl_ugni_repost (frag, OMPI_SUCCESS); return 0; } BTL_VERBOSE(("RDMA/FMA complete for frag %p", (void *) frag)); frag->cbfunc (frag, ompi_common_rc_ugni_to_ompi (rc)); return 1; }
static int __nic_rx_progress(struct gnix_nic *nic) { int ret = FI_SUCCESS; gni_return_t status = GNI_RC_NOT_DONE; gni_cq_entry_t cqe; status = GNI_CqTestEvent(nic->rx_cq); if (status == GNI_RC_NOT_DONE) return FI_SUCCESS; COND_ACQUIRE(nic->requires_lock, &nic->lock); do { status = GNI_CqGetEvent(nic->rx_cq, &cqe); if (unlikely(status == GNI_RC_NOT_DONE)) { ret = FI_SUCCESS; break; } if (likely(status == GNI_RC_SUCCESS)) { /* Find and schedule the associated VC. */ ret = __process_rx_cqe(nic, cqe); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_DATA, "process_rx_cqe() failed: %d\n", ret); } } else if (status == GNI_RC_ERROR_RESOURCE) { /* The remote CQ was overrun. Events related to any VC * could have been missed. Schedule each VC to be sure * all messages are processed. */ assert(GNI_CQ_OVERRUN(cqe)); __nic_rx_overrun(nic); } else { GNIX_WARN(FI_LOG_EP_DATA, "GNI_CqGetEvent returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); break; } } while (1); COND_RELEASE(nic->requires_lock, &nic->lock); return ret; }
void uct_ugni_progress(void *arg) { gni_cq_entry_t event_data = 0; gni_post_descriptor_t *event_post_desc_ptr; uct_ugni_base_desc_t *desc; uct_ugni_iface_t * iface = (uct_ugni_iface_t *)arg; gni_return_t ugni_rc; ugni_rc = GNI_CqGetEvent(iface->local_cq, &event_data); if (GNI_RC_NOT_DONE == ugni_rc) { goto out; } if ((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)) { ucs_error("GNI_CqGetEvent falied. Error status %s %d ", gni_err_str[ugni_rc], ugni_rc); return; } ugni_rc = GNI_GetCompleted(iface->local_cq, event_data, &event_post_desc_ptr); if (GNI_RC_SUCCESS != ugni_rc && GNI_RC_TRANSACTION_ERROR != ugni_rc) { ucs_error("GNI_GetCompleted falied. Error status %s %d %d", gni_err_str[ugni_rc], ugni_rc, GNI_RC_TRANSACTION_ERROR); return; } desc = (uct_ugni_base_desc_t *)event_post_desc_ptr; ucs_trace_async("Completion received on %p", desc); if (NULL != desc->comp_cb) { uct_invoke_completion(desc->comp_cb, UCS_OK); } --iface->outstanding; --desc->ep->outstanding; if (ucs_likely(0 == desc->not_ready_to_free)) { ucs_mpool_put(desc); } uct_ugni_ep_check_flush(desc->ep); out: /* have a go a processing the pending queue */ ucs_arbiter_dispatch(&iface->arbiter, 1, uct_ugni_ep_process_pending, NULL); return; }
int mca_btl_ugni_progress_remote_smsg (mca_btl_ugni_module_t *btl) { mca_btl_base_endpoint_t *ep; gni_cq_entry_t event_data; gni_return_t grc; uint64_t inst_id; grc = GNI_CqGetEvent (btl->smsg_remote_cq, &event_data); if (GNI_RC_NOT_DONE == grc) { return 0; } if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc || !GNI_CQ_STATUS_OK(event_data) || GNI_CQ_OVERRUN(event_data))) { if (GNI_RC_ERROR_RESOURCE == grc || (GNI_RC_SUCCESS == grc && GNI_CQ_OVERRUN(event_data))) { /* recover from smsg cq overrun */ return mca_btl_ugni_handle_remote_smsg_overrun (btl); } BTL_ERROR(("unhandled error in GNI_CqGetEvent")); /* unhandled error: crash */ assert (0); return ompi_common_rc_ugni_to_ompi (grc); } BTL_VERBOSE(("REMOTE CQ: Got event 0x%" PRIx64 ". msg id = %" PRIu64 ". ok = %d, type = %" PRIu64 "\n", (uint64_t) event_data, GNI_CQ_GET_MSG_ID(event_data), GNI_CQ_STATUS_OK(event_data), GNI_CQ_GET_TYPE(event_data))); inst_id = GNI_CQ_GET_INST_ID(event_data); ep = btl->endpoints[inst_id & 0xffffffff]; if (OPAL_UNLIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state)) { /* due to the nature of datagrams we may get a smsg completion before we get mailbox info from the peer */ BTL_VERBOSE(("event occurred on an unconnected endpoint! ep state = %d", ep->state)); return 0; } return mca_btl_ugni_smsg_process (ep); }
static void uct_ugni_smsg_handle_remote_overflow(uct_ugni_smsg_iface_t *iface){ gni_return_t ugni_rc; gni_cq_entry_t event_data; struct sglib_hashed_uct_ugni_ep_t_iterator ep_iterator; uct_ugni_ep_t *current_ep; uct_ugni_smsg_ep_t *ep; /* We don't know which EP dropped a completion entry, so flush everything */ do{ ugni_rc = GNI_CqGetEvent(iface->remote_cq, &event_data); } while(GNI_RC_NOT_DONE != ugni_rc); current_ep = sglib_hashed_uct_ugni_ep_t_it_init(&ep_iterator, iface->super.eps); while(NULL != current_ep){ ep = ucs_derived_of(current_ep, uct_ugni_smsg_ep_t); process_mbox(iface, ep); current_ep = sglib_hashed_uct_ugni_ep_t_it_next(&ep_iterator); } }
static int __nic_rx_overrun(struct gnix_nic *nic) { int i, max_id, ret; struct gnix_vc *vc; gni_return_t status; gni_cq_entry_t cqe; GNIX_WARN(FI_LOG_EP_DATA, "\n"); /* clear out the CQ */ /* * TODO: really need to process CQEs better for error reporting, * etc. */ while ((status = GNI_CqGetEvent(nic->rx_cq, &cqe)) == GNI_RC_SUCCESS); assert(status == GNI_RC_NOT_DONE); COND_ACQUIRE(nic->requires_lock, &nic->vc_id_lock); max_id = nic->vc_id_table_count; COND_RELEASE(nic->requires_lock, &nic->vc_id_lock); /* * TODO: optimization would * be to keep track of last time * this happened and where smsg msgs. * were found. */ for (i = 0; i < max_id; i++) { ret = _gnix_test_bit(&nic->vc_id_bitmap, i); if (ret) { vc = __gnix_nic_elem_by_rem_id(nic, i); ret = _gnix_vc_dequeue_smsg(vc); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_DATA, "_gnix_vc_dqueue_smsg returned %d\n", ret); } } } return FI_SUCCESS; }
void uct_ugni_progress(void *arg) { gni_cq_entry_t event_data = 0; gni_post_descriptor_t *event_post_desc_ptr; uct_ugni_base_desc_t *desc; uct_ugni_iface_t * iface = (uct_ugni_iface_t *)arg; gni_return_t ugni_rc; ugni_rc = GNI_CqGetEvent(iface->local_cq, &event_data); if (GNI_RC_NOT_DONE == ugni_rc) { return; } if ((GNI_RC_SUCCESS != ugni_rc && !event_data) || GNI_CQ_OVERRUN(event_data)) { ucs_error("GNI_CqGetEvent falied. Error status %s %d ", gni_err_str[ugni_rc], ugni_rc); return; } ugni_rc = GNI_GetCompleted(iface->local_cq, event_data, &event_post_desc_ptr); if (GNI_RC_SUCCESS != ugni_rc && GNI_RC_TRANSACTION_ERROR != ugni_rc) { ucs_error("GNI_GetCompleted falied. Error status %s %d %d", gni_err_str[ugni_rc], ugni_rc, GNI_RC_TRANSACTION_ERROR); return; } desc = (uct_ugni_base_desc_t *)event_post_desc_ptr; ucs_trace_async("Completion received on %p", desc); if (NULL != desc->comp_cb) { uct_invoke_completion(desc->comp_cb); } --iface->outstanding; --desc->ep->outstanding; if (ucs_likely(desc->not_ready_to_free == 0)) { ucs_mpool_put(desc); } return; }
static inline int mca_btl_ugni_progress_rdma (mca_btl_ugni_module_t *ugni_module, int which_cq) { mca_btl_ugni_post_descriptor_t *post_desc = NULL; gni_cq_entry_t event_data = 0; gni_post_descriptor_t *desc; uint32_t recoverable = 1; gni_return_t grc; gni_cq_handle_t the_cq; the_cq = (which_cq == 0) ? ugni_module->rdma_local_cq : ugni_module->rdma_local_irq_cq; OPAL_THREAD_LOCK(&ugni_module->device->dev_lock); grc = GNI_CqGetEvent (the_cq, &event_data); if (GNI_RC_NOT_DONE == grc) { OPAL_THREAD_UNLOCK(&ugni_module->device->dev_lock); return 0; } if (OPAL_UNLIKELY((GNI_RC_SUCCESS != grc && !event_data) || GNI_CQ_OVERRUN(event_data))) { /* TODO -- need to handle overrun -- how do we do this without an event? will the event eventually come back? Ask Cray */ BTL_ERROR(("unhandled post error! ugni rc = %d %s", grc, gni_err_str[grc])); OPAL_THREAD_UNLOCK(&ugni_module->device->dev_lock); return opal_common_rc_ugni_to_opal (grc); } grc = GNI_GetCompleted (the_cq, event_data, &desc); OPAL_THREAD_UNLOCK(&ugni_module->device->dev_lock); if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc && GNI_RC_TRANSACTION_ERROR != grc)) { BTL_ERROR(("Error in GNI_GetComplete %s", gni_err_str[grc])); return opal_common_rc_ugni_to_opal (grc); } post_desc = MCA_BTL_UGNI_DESC_TO_PDESC(desc); if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc || !GNI_CQ_STATUS_OK(event_data))) { (void) GNI_CqErrorRecoverable (event_data, &recoverable); if (OPAL_UNLIKELY(++post_desc->desc.tries >= mca_btl_ugni_component.rdma_max_retries || !recoverable)) { char char_buffer[1024]; GNI_CqErrorStr (event_data, char_buffer, 1024); /* give up */ BTL_ERROR(("giving up on desciptor %p, recoverable %d: %s", (void *) post_desc, recoverable, char_buffer)); #if OPAL_ENABLE_DEBUG btl_ugni_dump_post_desc (post_desc); #endif mca_btl_ugni_post_desc_complete (ugni_module, post_desc, OPAL_ERROR); return OPAL_ERROR; } mca_btl_ugni_repost (ugni_module, post_desc); return 0; } mca_btl_ugni_post_desc_complete (ugni_module, post_desc, opal_common_rc_ugni_to_opal (grc)); return 1; }
/* * this function is intended to be invoked as an argument to pthread_create, */ static void *__gnix_nic_prog_thread_fn(void *the_arg) { int ret = FI_SUCCESS, prev_state; int retry = 0; uint32_t which; struct gnix_nic *nic = (struct gnix_nic *)the_arg; sigset_t sigmask; gni_cq_handle_t cqv[2]; gni_return_t status; gni_cq_entry_t cqe; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); /* * temporarily disable cancelability while we set up * some stuff */ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &prev_state); /* * help out Cray core-spec, say we're not an app thread * and can be run on core-spec cpus. */ ret = _gnix_task_is_not_app(); if (ret) GNIX_WARN(FI_LOG_EP_CTRL, "_gnix_task_is_not_app call returned %d\n", ret); /* * block all signals, don't want this thread to catch * signals that may be for app threads */ memset(&sigmask, 0, sizeof(sigset_t)); ret = sigfillset(&sigmask); if (ret) { GNIX_WARN(FI_LOG_EP_CTRL, "sigfillset call returned %d\n", ret); } else { ret = pthread_sigmask(SIG_SETMASK, &sigmask, NULL); if (ret) GNIX_WARN(FI_LOG_EP_CTRL, "pthread_sigmask call returned %d\n", ret); } /* * okay now we're ready to be cancelable. */ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &prev_state); pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); cqv[0] = nic->tx_cq_blk; cqv[1] = nic->rx_cq_blk; try_again: status = GNI_CqVectorMonitor(cqv, 2, -1, &which); switch (status) { case GNI_RC_SUCCESS: /* * first dequeue RX CQEs */ if (which == 1) { do { status = GNI_CqGetEvent(nic->rx_cq_blk, &cqe); } while (status == GNI_RC_SUCCESS); } _gnix_nic_progress(nic); retry = 1; break; case GNI_RC_TIMEOUT: retry = 1; break; case GNI_RC_NOT_DONE: retry = 1; break; case GNI_RC_INVALID_PARAM: case GNI_RC_INVALID_STATE: case GNI_RC_ERROR_RESOURCE: case GNI_RC_ERROR_NOMEM: retry = 0; GNIX_WARN(FI_LOG_EP_CTRL, "GNI_CqGetEvent returned %s\n", gni_err_str[status]); break; default: retry = 0; GNIX_WARN(FI_LOG_EP_CTRL, "GNI_CqGetEvent returned unexpected code %s\n", gni_err_str[status]); break; } if (retry) goto try_again; return NULL; }