static ucs_status_t uct_ugni_add_flush_comp(uct_ugni_ep_t *ep, unsigned flags, uct_completion_t *comp) { uct_ugni_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_ugni_iface_t); uct_ugni_flush_group_t *new_group, *present_group; if (!uct_ugni_ep_can_send(ep)) { return UCS_ERR_NO_RESOURCE; } if (NULL == comp) { return UCS_INPROGRESS; } new_group = uct_ugni_new_flush_group(iface); new_group->flush_comp.count = UCT_UGNI_INIT_FLUSH_REQ; #ifdef DEBUG new_group->flush_comp.func = NULL; new_group->parent = NULL; #endif present_group = (uct_ugni_flush_group_t*)uct_ugni_safe_swap_pointers(&ep->flush_group, (uintptr_t)new_group); present_group->flush_comp.func = uct_ugni_flush_cb; present_group->user_comp = comp; present_group->parent = new_group; uct_invoke_completion(&present_group->flush_comp, UCS_OK); return UCS_INPROGRESS; }
static inline ucs_status_t uct_ugni_post_rdma(uct_ugni_rdma_iface_t *iface, uct_ugni_ep_t *ep, uct_ugni_base_desc_t *rdma) { gni_return_t ugni_rc; if (ucs_unlikely(!uct_ugni_ep_can_send(ep))) { ucs_mpool_put(rdma); return UCS_ERR_NO_RESOURCE; } uct_ugni_device_lock(&iface->super.cdm); ugni_rc = GNI_PostRdma(ep->ep, &rdma->desc); uct_ugni_device_unlock(&iface->super.cdm); if (ucs_unlikely(GNI_RC_SUCCESS != ugni_rc)) { ucs_mpool_put(rdma); if(GNI_RC_ERROR_RESOURCE == ugni_rc || GNI_RC_ERROR_NOMEM == ugni_rc) { ucs_debug("GNI_PostRdma failed, Error status: %s %d", gni_err_str[ugni_rc], ugni_rc); return UCS_ERR_NO_RESOURCE; } else { ucs_error("GNI_PostRdma failed, Error status: %s %d", gni_err_str[ugni_rc], ugni_rc); return UCS_ERR_IO_ERROR; } } ++rdma->flush_group->flush_comp.count; ++iface->super.outstanding; return UCS_INPROGRESS; }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_ugni_smsg_ep_am_common_send(uct_ugni_smsg_ep_t *ep, uct_ugni_smsg_iface_t *iface, uint8_t am_id, unsigned header_length, void *header, unsigned payload_length, void *payload, uct_ugni_smsg_desc_t *desc) { gni_return_t gni_rc; if (ucs_unlikely(!uct_ugni_ep_can_send(&ep->super))) { goto exit_no_res; } desc->msg_id = iface->smsg_id++; desc->flush_group = ep->super.flush_group; uct_ugni_cdm_lock(&iface->super.cdm); gni_rc = GNI_SmsgSendWTag(ep->super.ep, header, header_length, payload, payload_length, desc->msg_id, am_id); uct_ugni_cdm_unlock(&iface->super.cdm); if(GNI_RC_SUCCESS != gni_rc){ goto exit_no_res; } ++desc->flush_group->flush_comp.count; ++iface->super.outstanding; sglib_hashed_uct_ugni_smsg_desc_t_add(iface->smsg_list, desc); return UCS_OK; exit_no_res: ucs_trace("Smsg send failed."); ucs_mpool_put(desc); UCS_STATS_UPDATE_COUNTER(ep->super.super.stats, UCT_EP_STAT_NO_RES, 1); return UCS_ERR_NO_RESOURCE; }