void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
{
struct mlx4_cqe *cqe, *dest;
uint32_t prod_index;
uint8_t owner_bit;
int nfreed = 0;
int is_xrc_srq = 0;
int cqe_inc = cq->cqe_size == 64 ? 1 : 0;
if (srq && srq->ibv_srq.xrc_cq)
is_xrc_srq = 1;
/*
* First we need to find the current producer index, so we
* know where to start cleaning from. It doesn't matter if HW
* adds new entries after this loop -- the QP we're worried
* about is already in RESET, so the new entries won't come
* from our QP and therefore don't need to be checked.
*/
for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
break;
/*
* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
while ((int) --prod_index - (int) cq->cons_index >= 0) {
cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
cqe += cqe_inc;
if (is_xrc_srq &&
(ntohl(cqe->g_mlpath_rqpn & 0xffffff) == srq->srqn) &&
!(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) {
mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
++nfreed;
} else if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) {
if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
++nfreed;
} else if (nfreed) {
dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe);
dest += cqe_inc;
owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
memcpy(dest, cqe, sizeof *cqe);
dest->owner_sr_opcode = owner_bit |
(dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
}
}
if (nfreed) {
cq->cons_index += nfreed;
/*
* Make sure update of buffer contents is done before
* updating consumer index.
*/
wmb();
update_cons_index(cq);
}
}
static int mlx4_poll_one(struct mlx4_cq *cq,
struct mlx4_qp **cur_qp,
struct ibv_wc *wc)
{
struct mlx4_wq *wq;
struct mlx4_cqe *cqe;
struct mlx4_srq *srq = NULL;
uint32_t qpn;
uint32_t srqn;
uint32_t g_mlpath_rqpn;
uint16_t wqe_index;
int is_error;
int is_send;
cqe = next_cqe_sw(cq);
if (!cqe)
return CQ_EMPTY;
++cq->cons_index;
VALGRIND_MAKE_MEM_DEFINED(cqe, sizeof *cqe);
/*
* Make sure we read CQ entry contents after we've checked the
* ownership bit.
*/
rmb();
qpn = ntohl(cqe->my_qpn);
is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR;
if (qpn & MLX4_XRC_QPN_BIT && !is_send) {
srqn = ntohl(cqe->g_mlpath_rqpn) & 0xffffff;
/*
* We do not have to take the XRC SRQ table lock here,
* because CQs will be locked while XRC SRQs are removed
* from the table.
*/
srq = mlx4_find_xrc_srq(to_mctx(cq->ibv_cq.context), srqn);
if (!srq)
return CQ_POLL_ERR;
} else if (!*cur_qp || (qpn & 0xffffff) != (*cur_qp)->ibv_qp.qp_num) {
/*
* We do not have to take the QP table lock here,
* because CQs will be locked while QPs are removed
* from the table.
*/
*cur_qp = mlx4_find_qp(to_mctx(cq->ibv_cq.context),
qpn & 0xffffff);
if (!*cur_qp)
return CQ_POLL_ERR;
}
wc->qp_num = qpn & 0xffffff;
if (is_send) {
wq = &(*cur_qp)->sq;
wqe_index = ntohs(cqe->wqe_index);
wq->tail += (uint16_t) (wqe_index - (uint16_t) wq->tail);
wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
++wq->tail;
} else if (srq) {
wqe_index = htons(cqe->wqe_index);
wc->wr_id = srq->wrid[wqe_index];
mlx4_free_srq_wqe(srq, wqe_index);
} else if ((*cur_qp)->ibv_qp.srq) {
srq = to_msrq((*cur_qp)->ibv_qp.srq);
wqe_index = htons(cqe->wqe_index);
wc->wr_id = srq->wrid[wqe_index];
mlx4_free_srq_wqe(srq, wqe_index);
} else {
wq = &(*cur_qp)->rq;
wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
++wq->tail;
}
if (is_error) {
mlx4_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);
return CQ_OK;
}
wc->status = IBV_WC_SUCCESS;
if (is_send) {
wc->wc_flags = 0;
switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
case MLX4_OPCODE_RDMA_WRITE_IMM:
wc->wc_flags |= IBV_WC_WITH_IMM;
case MLX4_OPCODE_RDMA_WRITE:
wc->opcode = IBV_WC_RDMA_WRITE;
break;
case MLX4_OPCODE_SEND_IMM:
wc->wc_flags |= IBV_WC_WITH_IMM;
case MLX4_OPCODE_SEND:
wc->opcode = IBV_WC_SEND;
break;
case MLX4_OPCODE_RDMA_READ:
wc->opcode = IBV_WC_RDMA_READ;
wc->byte_len = ntohl(cqe->byte_cnt);
break;
case MLX4_OPCODE_ATOMIC_CS:
wc->opcode = IBV_WC_COMP_SWAP;
wc->byte_len = 8;
break;
case MLX4_OPCODE_ATOMIC_FA:
wc->opcode = IBV_WC_FETCH_ADD;
wc->byte_len = 8;
break;
case MLX4_OPCODE_BIND_MW:
wc->opcode = IBV_WC_BIND_MW;
break;
default:
/* assume it's a send completion */
wc->opcode = IBV_WC_SEND;
break;
}
} else {
wc->byte_len = ntohl(cqe->byte_cnt);
switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
wc->wc_flags = IBV_WC_WITH_IMM;
wc->imm_data = cqe->immed_rss_invalid;
break;
case MLX4_RECV_OPCODE_SEND:
wc->opcode = IBV_WC_RECV;
wc->wc_flags = 0;
break;
case MLX4_RECV_OPCODE_SEND_IMM:
wc->opcode = IBV_WC_RECV;
wc->wc_flags = IBV_WC_WITH_IMM;
wc->imm_data = cqe->immed_rss_invalid;
break;
}
wc->slid = ntohs(cqe->rlid);
wc->sl = cqe->sl >> 4;
g_mlpath_rqpn = ntohl(cqe->g_mlpath_rqpn);
wc->src_qp = g_mlpath_rqpn & 0xffffff;
wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f;
wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IBV_WC_GRH : 0;
wc->pkey_index = ntohl(cqe->immed_rss_invalid) & 0x7f;
}
return CQ_OK;
}
