ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); ucs_status_t status; if (!uct_rc_iface_has_tx_resources(&iface->super)) { return UCS_ERR_NO_RESOURCE; } if (uct_rc_txqp_available(&ep->super.txqp) == iface->config.tx_max_wr) { UCT_TL_EP_STAT_FLUSH(&ep->super.super); return UCS_OK; } if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) { if (IBV_DEVICE_HAS_NOP(&uct_ib_iface_device(&iface->super.super)->dev_attr)) { status = uct_rc_verbs_ep_nop(ep); } else { status = uct_rc_verbs_ep_put_short(tl_ep, NULL, 0, 0, 0); } if (status != UCS_OK) { return status; } } else if (!uct_rc_ep_has_tx_resources(&ep->super)) { return UCS_ERR_NO_RESOURCE; } uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, ep->txcnt.pi); UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super); return UCS_INPROGRESS; }
ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uint16_t sn; if (!uct_rc_iface_has_tx_resources(&iface->super)) { return UCS_ERR_NO_RESOURCE; } if (uct_rc_txqp_available(&ep->super.txqp) == ep->tx.wq.bb_max) { UCT_TL_EP_STAT_FLUSH(&ep->super.super); return UCS_OK; } if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) { sn = ep->tx.wq.sw_pi; UCT_RC_CHECK_RES(&iface->super, &ep->super); uct_rc_mlx5_txqp_inline_post(&iface->super, IBV_QPT_RC, &ep->super.txqp, &ep->tx.wq, MLX5_OPCODE_NOP, NULL, 0, 0, 0, 0, 0, NULL, 0); } else if (!uct_rc_ep_has_tx_resources(&ep->super)) { return UCS_ERR_NO_RESOURCE; } else { sn = ep->tx.wq.sig_pi; } uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, sn); UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super); return UCS_INPROGRESS; }
ucs_status_t uct_cm_ep_flush(uct_ep_h tl_ep) { ucs_status_t status; status = uct_cm_iface_flush_do(tl_ep->iface); if (status == UCS_OK) { UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t)); } else { UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); } return status; }
static ucs_status_t uct_ugni_smsg_ep_flush(uct_ep_h tl_ep){ uct_ugni_smsg_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_smsg_ep_t); uct_ugni_smsg_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ugni_smsg_iface_t); if (0 == ep->super.outstanding) { UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t)); return UCS_OK; } progress_local_cq(iface); UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); return UCS_INPROGRESS; }
ucs_status_t uct_ugni_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_ugni_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_ep_t); ucs_status_t status = UCS_INPROGRESS; ucs_trace_func("tl_ep=%p, flags=%x, comp=%p", tl_ep, flags, comp); if (uct_ugni_ep_can_flush(ep)) { UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t)); return UCS_OK; } status = uct_ugni_add_flush_comp(ep, flags, comp); if (UCS_INPROGRESS == status) { UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); } return status; }
ucs_status_t uct_ugni_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_ugni_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_ep_t); uct_ugni_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ugni_iface_t); if (uct_ugni_can_flush(ep)) { UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t)); return UCS_OK; } ep->flush_flag = 1; ep->arb_flush = ep->arb_size; uct_ugni_progress(iface); UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); return UCS_ERR_NO_RESOURCE; }
ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep) { uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); ucs_status_t status; if (ep->super.available == ep->tx.wq.bb_max) { UCT_TL_EP_STAT_FLUSH(&ep->super.super); ucs_trace_data("ep %p is flushed", ep); return UCS_OK; } if (ep->super.unsignaled != 0) { status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_NOP, NULL, 0, 0, 0, 0, 0); if (status != UCS_OK) { return status; } } UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super); return UCS_INPROGRESS; }
ucs_status_t uct_dc_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_dc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_iface_t); uct_dc_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_ep_t); ucs_status_t status; /* If waiting for FC grant, return NO_RESOURCE to prevent ep destruction. * Otherwise grant for destroyed ep will arrive and there will be a * segfault when we will try to access the ep by address from the grant * message. */ if (!uct_rc_iface_has_tx_resources(&iface->super)) { return UCS_ERR_NO_RESOURCE; } if (ep->dci == UCT_DC_EP_NO_DCI) { if (!uct_dc_iface_dci_can_alloc(iface)) { return UCS_ERR_NO_RESOURCE; /* waiting for dci */ } else { UCT_TL_EP_STAT_FLUSH(&ep->super); /* no sends */ return UCS_OK; } } if (!uct_dc_iface_dci_ep_can_send(ep)) { return UCS_ERR_NO_RESOURCE; /* cannot send */ } status = uct_dc_iface_flush_dci(iface, ep->dci); if (status == UCS_OK) { UCT_TL_EP_STAT_FLUSH(&ep->super); return UCS_OK; /* all sends completed */ } ucs_assert(status == UCS_INPROGRESS); UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super); return UCS_INPROGRESS; }
static ucs_status_t uct_ugni_smsg_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_ugni_smsg_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_smsg_ep_t); ucs_status_t status; /* Even if we have no outstanding requests we can still get back send credits */ status = progress_local_cq(ucs_derived_of(tl_ep->iface, uct_ugni_smsg_iface_t)); if((0 == ep->super.outstanding) && (ep->super.arb_size == 0)) { ep->super.flush_flag = 0; } if(uct_ugni_can_flush(&ep->super)) { UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t)); status = UCS_OK; } else { ep->super.flush_flag = 1; UCT_TL_EP_STAT_FLUSH_WAIT(ucs_derived_of(tl_ep, uct_base_ep_t)); status = UCS_ERR_NO_RESOURCE; } return status; }