ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
ucs_status_t status;
if (!uct_rc_iface_has_tx_resources(&iface->super)) {
return UCS_ERR_NO_RESOURCE;
}
if (uct_rc_txqp_available(&ep->super.txqp) == iface->config.tx_max_wr) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
return UCS_OK;
}
if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) {
if (IBV_DEVICE_HAS_NOP(&uct_ib_iface_device(&iface->super.super)->dev_attr)) {
status = uct_rc_verbs_ep_nop(ep);
} else {
status = uct_rc_verbs_ep_put_short(tl_ep, NULL, 0, 0, 0);
}
if (status != UCS_OK) {
return status;
}
} else if (!uct_rc_ep_has_tx_resources(&ep->super)) {
return UCS_ERR_NO_RESOURCE;
}
uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, ep->txcnt.pi);
UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super);
return UCS_INPROGRESS;
}
static UCS_F_ALWAYS_INLINE void
uct_rc_verbs_exp_post_send(uct_rc_verbs_ep_t *ep, struct ibv_exp_send_wr *wr,
uint64_t signal)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_verbs_iface_t);
uct_rc_txqp_check(&ep->super.txqp);
struct ibv_exp_send_wr *bad_wr;
int ret;
signal |= uct_rc_iface_tx_moderation(&iface->super, &ep->super.txqp,
IBV_EXP_SEND_SIGNALED);
wr->exp_send_flags = signal;
wr->wr_id = uct_rc_txqp_unsignaled(&ep->super.txqp);
uct_ib_log_exp_post_send(&iface->super.super, ep->super.txqp.qp, wr,
(wr->exp_opcode == IBV_EXP_WR_SEND) ? uct_rc_ep_am_packet_dump : NULL);
UCT_IB_INSTRUMENT_RECORD_SEND_EXP_WR_LEN("uct_rc_verbs_exp_post_send", wr);
ret = ibv_exp_post_send(ep->super.txqp.qp, wr, &bad_wr);
if (ret != 0) {
ucs_fatal("ibv_exp_post_send() returned %d (%m)", ret);
}
uct_rc_verbs_txqp_posted(&ep->super.txqp, &ep->txcnt, &iface->super, signal);
}
static UCS_F_ALWAYS_INLINE void
uct_rc_verbs_ep_post_send(uct_rc_verbs_iface_t* iface, uct_rc_verbs_ep_t* ep,
struct ibv_send_wr *wr, int send_flags)
{
struct ibv_send_wr *bad_wr;
int ret;
uct_rc_txqp_check(&ep->super.txqp);
if (!(send_flags & IBV_SEND_SIGNALED)) {
send_flags |= uct_rc_iface_tx_moderation(&iface->super, &ep->super.txqp,
IBV_SEND_SIGNALED);
}
wr->send_flags = send_flags;
wr->wr_id = uct_rc_txqp_unsignaled(&ep->super.txqp);
uct_ib_log_post_send(&iface->super.super, ep->super.txqp.qp, wr,
(wr->opcode == IBV_WR_SEND) ? uct_rc_ep_am_packet_dump : NULL);
UCT_IB_INSTRUMENT_RECORD_SEND_WR_LEN("uct_rc_verbs_ep_post_send", wr);
ret = ibv_post_send(ep->super.txqp.qp, wr, &bad_wr);
if (ret != 0) {
ucs_fatal("ibv_post_send() returned %d (%m)", ret);
}
uct_rc_verbs_txqp_posted(&ep->super.txqp, &ep->txcnt, &iface->super, send_flags & IBV_SEND_SIGNALED);
}
ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
ucs_status_t status;
if (ucs_unlikely(flags & UCT_FLUSH_FLAG_CANCEL)) {
uct_ep_pending_purge(&ep->super.super.super, NULL, 0);
uct_rc_verbs_ep_handle_failure(ep, UCS_ERR_CANCELED);
return UCS_OK;
}
status = uct_rc_ep_flush(&ep->super, iface->config.tx_max_wr, flags);
if (status != UCS_INPROGRESS) {
return status;
}
if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) {
status = uct_rc_verbs_ep_put_short(tl_ep, NULL, 0, 0, 0);
if (status != UCS_OK) {
return status;
}
}
return uct_rc_txqp_add_flush_comp(&iface->super, &ep->super.super,
&ep->super.txqp, comp, ep->txcnt.pi);
}
static UCS_F_ALWAYS_INLINE void
uct_rc_verbs_ep_post_send(uct_rc_verbs_iface_t* iface, uct_rc_verbs_ep_t* ep,
struct ibv_send_wr *wr, int send_flags, int max_log_sge)
{
struct ibv_send_wr *bad_wr;
int ret;
uct_rc_txqp_check(&ep->super.txqp);
if (!(send_flags & IBV_SEND_SIGNALED)) {
send_flags |= uct_rc_iface_tx_moderation(&iface->super, &ep->super.txqp,
IBV_SEND_SIGNALED);
}
if (wr->opcode == IBV_WR_RDMA_READ) {
send_flags |= uct_rc_ep_atomic_fence(&iface->super, &ep->fi,
IBV_SEND_FENCE);
}
wr->send_flags = send_flags;
wr->wr_id = uct_rc_txqp_unsignaled(&ep->super.txqp);
uct_ib_log_post_send(&iface->super.super, ep->super.txqp.qp, wr, max_log_sge,
(wr->opcode == IBV_WR_SEND) ? uct_rc_ep_packet_dump : NULL);
ret = ibv_post_send(ep->super.txqp.qp, wr, &bad_wr);
if (ret != 0) {
ucs_fatal("ibv_post_send() returned %d (%m)", ret);
}
uct_rc_verbs_txqp_posted(&ep->super.txqp, &ep->txcnt, &iface->super, send_flags & IBV_SEND_SIGNALED);
}
ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags,
uct_completion_t *comp)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uint16_t sn;
if (!uct_rc_iface_has_tx_resources(&iface->super)) {
return UCS_ERR_NO_RESOURCE;
}
if (uct_rc_txqp_available(&ep->super.txqp) == ep->tx.wq.bb_max) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
return UCS_OK;
}
if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) {
sn = ep->tx.wq.sw_pi;
UCT_RC_CHECK_RES(&iface->super, &ep->super);
uct_rc_mlx5_txqp_inline_post(&iface->super, IBV_QPT_RC,
&ep->super.txqp, &ep->tx.wq,
MLX5_OPCODE_NOP, NULL, 0,
0, 0,
0, 0,
NULL, 0);
} else if (!uct_rc_ep_has_tx_resources(&ep->super)) {
return UCS_ERR_NO_RESOURCE;
} else {
sn = ep->tx.wq.sig_pi;
}
uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, sn);
UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super);
return UCS_INPROGRESS;
}
