void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
{
struct mlx4_cqe *cqe, *dest;
uint32_t prod_index;
uint8_t owner_bit;
int nfreed = 0;
int is_xrc_srq = 0;
int cqe_inc = cq->cqe_size == 64 ? 1 : 0;
if (srq && srq->ibv_srq.xrc_cq)
is_xrc_srq = 1;
/*
* First we need to find the current producer index, so we
* know where to start cleaning from. It doesn't matter if HW
* adds new entries after this loop -- the QP we're worried
* about is already in RESET, so the new entries won't come
* from our QP and therefore don't need to be checked.
*/
for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
break;
/*
* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
while ((int) --prod_index - (int) cq->cons_index >= 0) {
cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
cqe += cqe_inc;
if (is_xrc_srq &&
(ntohl(cqe->g_mlpath_rqpn & 0xffffff) == srq->srqn) &&
!(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) {
mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
++nfreed;
} else if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) {
if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
++nfreed;
} else if (nfreed) {
dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe);
dest += cqe_inc;
owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
memcpy(dest, cqe, sizeof *cqe);
dest->owner_sr_opcode = owner_bit |
(dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
}
}
if (nfreed) {
cq->cons_index += nfreed;
/*
* Make sure update of buffer contents is done before
* updating consumer index.
*/
wmb();
update_cons_index(cq);
}
}
int mlx4_get_outstanding_cqes(struct mlx4_cq *cq)
{
uint32_t i;
for (i = cq->cons_index; get_sw_cqe(cq, (i & cq->ibv_cq.cqe)); ++i)
;
return i - cq->cons_index;
}
static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq)
{
u32 i;
i = cq->mcq.cons_index;
while (get_sw_cqe(cq, i & cq->ibcq.cqe))
++i;
return i - cq->mcq.cons_index;
}
void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
{
u32 prod_index;
int nfreed = 0;
struct mlx4_cqe *cqe, *dest;
u8 owner_bit;
/*
* First we need to find the current producer index, so we
* know where to start cleaning from. It doesn't matter if HW
* adds new entries after this loop -- the QP we're worried
* about is already in RESET, so the new entries won't come
* from our QP and therefore don't need to be checked.
*/
for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
break;
/*
* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
if ((be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) {
if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));
++nfreed;
} else if (nfreed) {
dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
memcpy(dest, cqe, sizeof *cqe);
dest->owner_sr_opcode = owner_bit |
(dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
}
}
if (nfreed) {
cq->mcq.cons_index += nfreed;
/*
* Make sure update of buffer contents is done before
* updating consumer index.
*/
wmb();
mlx4_cq_set_ci(&cq->mcq);
}
}
static void __hns_roce_v1_cq_clean(struct hns_roce_cq *cq, uint32_t qpn,
struct hns_roce_srq *srq)
{
int nfreed = 0;
uint32_t prod_index;
uint8_t owner_bit = 0;
struct hns_roce_cqe *cqe, *dest;
struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context);
for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index);
++prod_index)
if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
break;
while ((int) --prod_index - (int) cq->cons_index >= 0) {
cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
CQE_BYTE_16_LOCAL_QPN_S) & 0xffffff) == qpn) {
++nfreed;
} else if (nfreed) {
dest = get_cqe(cq,
(prod_index + nfreed) & cq->ibv_cq.cqe);
owner_bit = roce_get_bit(dest->cqe_byte_4,
CQE_BYTE_4_OWNER_S);
memcpy(dest, cqe, sizeof(*cqe));
roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
owner_bit);
}
}
if (nfreed) {
cq->cons_index += nfreed;
udma_to_device_barrier();
hns_roce_update_cq_cons_index(ctx, cq);
}
}
static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)
{
return get_sw_cqe(cq, cq->mcq.cons_index);
}
static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *cq)
{
return get_sw_cqe(cq, cq->cons_index);
}
