ucs_status_t progress_remote_cq(uct_ugni_smsg_iface_t *iface)
{
gni_return_t ugni_rc;
gni_cq_entry_t event_data;
uct_ugni_ep_t tl_ep;
uct_ugni_ep_t *ugni_ep;
uct_ugni_smsg_ep_t *ep;
ugni_rc = GNI_CqGetEvent(iface->remote_cq, &event_data);
if(GNI_RC_NOT_DONE == ugni_rc){
return UCS_OK;
}
if (GNI_RC_SUCCESS != ugni_rc || !GNI_CQ_STATUS_OK(event_data) || GNI_CQ_OVERRUN(event_data)) {
if(GNI_RC_ERROR_RESOURCE == ugni_rc || (GNI_RC_SUCCESS == ugni_rc && GNI_CQ_OVERRUN(event_data))){
ucs_debug("Detected remote CQ overrun. ungi_rc = %d [%s]", ugni_rc, gni_err_str[ugni_rc]);
uct_ugni_smsg_handle_remote_overflow(iface);
return UCS_OK;
}
ucs_error("GNI_CqGetEvent falied with unhandled error. Error status %s %d ",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
tl_ep.hash_key = GNI_CQ_GET_INST_ID(event_data);
ugni_ep = sglib_hashed_uct_ugni_ep_t_find_member(iface->super.eps, &tl_ep);
ep = ucs_derived_of(ugni_ep, uct_ugni_smsg_ep_t);
process_mbox(iface, ep);
return UCS_INPROGRESS;
}
int mca_btl_ugni_progress_remote_smsg (mca_btl_ugni_module_t *btl)
{
mca_btl_base_endpoint_t *ep;
gni_cq_entry_t event_data;
gni_return_t grc;
uint64_t inst_id;
grc = mca_btl_ugni_gni_cq_get_event (btl->devices, btl->smsg_remote_cq, &event_data);
if (GNI_RC_NOT_DONE == grc) {
return 0;
}
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc || !GNI_CQ_STATUS_OK(event_data) ||
GNI_CQ_OVERRUN(event_data))) {
if (GNI_RC_ERROR_RESOURCE == grc ||
(GNI_RC_SUCCESS == grc && GNI_CQ_OVERRUN(event_data))) {
/* recover from smsg cq overrun */
return mca_btl_ugni_handle_remote_smsg_overrun (btl);
}
BTL_ERROR(("unhandled error in GNI_CqGetEvent"));
/* unhandled error: crash */
assert (0);
return mca_btl_rc_ugni_to_opal (grc);
}
BTL_VERBOSE(("REMOTE CQ: Got event 0x%" PRIx64 ". msg id = %" PRIu64
". ok = %d, type = %" PRIu64, (uint64_t) event_data,
GNI_CQ_GET_INST_ID(event_data), GNI_CQ_STATUS_OK(event_data),
GNI_CQ_GET_TYPE(event_data)));
inst_id = GNI_CQ_GET_INST_ID(event_data);
ep = (mca_btl_base_endpoint_t *) opal_pointer_array_get_item (&btl->endpoints, inst_id);
if (OPAL_UNLIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTED != ep->state)) {
/* due to the nature of datagrams we may get a smsg completion before
we get mailbox info from the peer */
BTL_VERBOSE(("event occurred on an unconnected endpoint! ep state = %d", ep->state));
return 0;
}
return mca_btl_ugni_smsg_process (ep);
}
static int __process_rx_cqe(struct gnix_nic *nic, gni_cq_entry_t cqe)
{
int ret = FI_SUCCESS, vc_id = 0;
struct gnix_vc *vc;
vc_id = GNI_CQ_GET_INST_ID(cqe);
/*
* its possible this vc has been destroyed, so may get NULL
* back.
*/
vc = __gnix_nic_elem_by_rem_id(nic, vc_id);
if (vc != NULL) {
switch (vc->conn_state) {
case GNIX_VC_CONNECTING:
GNIX_DEBUG(FI_LOG_EP_DATA,
"Scheduling VC for RX processing (%p)\n",
vc);
ret = _gnix_vc_rx_schedule(vc);
assert(ret == FI_SUCCESS);
break;
case GNIX_VC_CONNECTED:
GNIX_DEBUG(FI_LOG_EP_DATA,
"Processing VC RX (%p)\n",
vc);
ret = _gnix_vc_dequeue_smsg(vc);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_vc_dqueue_smsg returned %d\n",
ret);
}
break;
default:
break; /* VC not in a state for scheduling or
SMSG processing */
}
}
return ret;
}
