ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
uint16_t exp_max_pi;
/*
* If we got completion for the last posted WQE, max_pi would be advanced
* to the value calculated from prev_sw_pi - which is the index where the last
* posted WQE started. See also uct_rc_mlx5_iface_poll_tx().
*/
exp_max_pi = uct_rc_mlx5_calc_max_pi(iface, ep->tx.prev_sw_pi);
if (ep->tx.max_pi == exp_max_pi) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
ucs_trace_data("ep %p is flushed", ep);
return UCS_OK;
}
if (ep->super.unsignaled != 0) {
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_NOP, NULL, 0, 0, 0, 0, 0);
if (status != UCS_OK) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
return status;
}
}
return UCS_INPROGRESS;
}
ucs_status_t uct_rc_mlx5_ep_put_short(uct_ep_h tl_ep, const void *buffer, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_RDMA_WRITE, buffer,
length, 0, 0, remote_addr, rkey);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, PUT, SHORT, length);
return status;
}
ucs_status_t uct_rc_mlx5_ep_fc_ctrl(uct_rc_ep_t *rc_ep)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(rc_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_SEND|
UCT_RC_MLX5_OPCODE_FLAG_RAW, NULL, 0,
UCT_RC_EP_FC_PURE_GRANT, 0, 0, 0);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, AM, SHORT, 0);
return status;
}
ucs_status_t uct_rc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *payload, unsigned length)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
UCT_CHECK_AM_ID(id);
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_SEND, payload, length,
id, hdr, 0, 0);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, AM, SHORT,
sizeof(hdr) + length);
return status;
}
ucs_status_t uct_rc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *payload, unsigned length)
{
uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
UCT_CHECK_AM_ID(id);
UCT_RC_CHECK_FC_WND(iface, &ep->super, id);
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_SEND, payload, length,
id, hdr, 0, 0);
if (ucs_likely(status >= 0)) {
UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length);
UCT_RC_UPDATE_FC_WND(&ep->super);
}
return status;
}
ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
if (ep->super.available == ep->tx.wq.bb_max) {
UCT_TL_EP_STAT_FLUSH(&ep->super.super);
ucs_trace_data("ep %p is flushed", ep);
return UCS_OK;
}
if (ep->super.unsignaled != 0) {
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_NOP, NULL, 0, 0, 0, 0, 0);
if (status != UCS_OK) {
return status;
}
}
UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super);
return UCS_INPROGRESS;
}
