static inline ucs_status_t uct_ud_verbs_iface_poll_rx(uct_ud_verbs_iface_t *iface)
{
uct_ib_iface_recv_desc_t *desc;
struct ibv_wc wc[UCT_IB_MAX_WC];
int i, ret;
char *packet;
ret = ibv_poll_cq(iface->super.super.recv_cq, UCT_IB_MAX_WC, wc);
if (ret == 0) {
return UCS_ERR_NO_PROGRESS;
}
if (ucs_unlikely(ret < 0)) {
ucs_fatal("Failed to poll receive CQ");
}
for (i = 0; i < ret; ++i) {
if (ucs_unlikely(wc[i].status != IBV_WC_SUCCESS)) {
ucs_fatal("Receive completion with error: %s", ibv_wc_status_str(wc[i].status));
}
desc = (void*)wc[i].wr_id;
ucs_trace_data("pkt rcvd: buf=%p len=%d", desc, wc[i].byte_len);
packet = uct_ib_iface_recv_desc_hdr(&iface->super.super, desc);
VALGRIND_MAKE_MEM_DEFINED(packet, wc[i].byte_len);
uct_ud_ep_process_rx(&iface->super,
(uct_ud_neth_t *)(packet + UCT_IB_GRH_LEN),
wc[i].byte_len - UCT_IB_GRH_LEN,
(uct_ud_recv_skb_t *)desc);
}
iface->super.rx.available += ret;
uct_ud_verbs_iface_post_recv(iface);
return UCS_OK;
}
static UCS_F_ALWAYS_INLINE
ucs_status_t uct_ud_mlx5_iface_poll_rx(uct_ud_mlx5_iface_t *iface)
{
struct mlx5_cqe64 *cqe;
uint16_t ci;
uct_ib_iface_recv_desc_t *desc;
uint32_t len;
void *packet;
ucs_status_t status;
ci = iface->rx.wq.cq_wqe_counter & iface->rx.wq.mask;
packet = (void *)ntohll(iface->rx.wq.wqes[ci].addr);
ucs_prefetch(packet + UCT_IB_GRH_LEN);
desc = (uct_ib_iface_recv_desc_t *)(packet - iface->super.super.config.rx_hdr_offset);
cqe = uct_ib_mlx5_get_cqe(&iface->rx.cq, UCT_IB_MLX5_CQE64_SIZE_LOG);
if (cqe == NULL) {
status = UCS_ERR_NO_PROGRESS;
goto out;
}
uct_ib_mlx5_log_cqe(cqe);
ucs_assert(0 == (cqe->op_own &
(MLX5_INLINE_SCATTER_32|MLX5_INLINE_SCATTER_64)));
ucs_assert(ntohs(cqe->wqe_counter) == iface->rx.wq.cq_wqe_counter);
iface->super.rx.available++;
iface->rx.wq.cq_wqe_counter++;
len = ntohl(cqe->byte_cnt);
VALGRIND_MAKE_MEM_DEFINED(packet, len);
uct_ud_ep_process_rx(&iface->super,
(uct_ud_neth_t *)(packet + UCT_IB_GRH_LEN),
len - UCT_IB_GRH_LEN,
(uct_ud_recv_skb_t *)desc);
status = UCS_OK;
out:
if (iface->super.rx.available >= iface->super.config.rx_max_batch) {
/* we need to try to post buffers always. Otherwise it is possible
* to run out of rx wqes if receiver is slow and there are always
* cqe to process
*/
uct_ud_mlx5_iface_post_recv(iface);
}
return status;
}
