void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
{
struct mlx4_cqe *cqe, *dest;
uint32_t prod_index;
uint8_t owner_bit;
int nfreed = 0;
int is_xrc_srq = 0;
int cqe_inc = cq->cqe_size == 64 ? 1 : 0;
if (srq && srq->ibv_srq.xrc_cq)
is_xrc_srq = 1;
/*
* First we need to find the current producer index, so we
* know where to start cleaning from. It doesn't matter if HW
* adds new entries after this loop -- the QP we're worried
* about is already in RESET, so the new entries won't come
* from our QP and therefore don't need to be checked.
*/
for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
break;
/*
* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
while ((int) --prod_index - (int) cq->cons_index >= 0) {
cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
cqe += cqe_inc;
if (is_xrc_srq &&
(ntohl(cqe->g_mlpath_rqpn & 0xffffff) == srq->srqn) &&
!(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) {
mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
++nfreed;
} else if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) {
if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
++nfreed;
} else if (nfreed) {
dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe);
dest += cqe_inc;
owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
memcpy(dest, cqe, sizeof *cqe);
dest->owner_sr_opcode = owner_bit |
(dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
}
}
if (nfreed) {
cq->cons_index += nfreed;
/*
* Make sure update of buffer contents is done before
* updating consumer index.
*/
wmb();
update_cons_index(cq);
}
}
static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq)
{
struct mlx4_cqe *cqe;
int i;
i = cq->mcq.cons_index;
cqe = get_cqe(cq, i & cq->ibcq.cqe);
while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {
memcpy(get_cqe_from_buf(&cq->resize_buf->buf,
(i + 1) & cq->resize_buf->cqe),
get_cqe(cq, i & cq->ibcq.cqe), sizeof(struct mlx4_cqe));
cqe = get_cqe(cq, ++i & cq->ibcq.cqe);
}
++cq->mcq.cons_index;
}
static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)
{
struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);
return (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
!!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;
}
static void *get_sw_cqe(struct hns_roce_cq *cq, int n)
{
struct hns_roce_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe);
return (!!(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S)) ^
!!(n & (cq->ibv_cq.cqe + 1))) ? cqe : NULL;
}
void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
{
u32 prod_index;
int nfreed = 0;
struct mlx4_cqe *cqe, *dest;
u8 owner_bit;
/*
* First we need to find the current producer index, so we
* know where to start cleaning from. It doesn't matter if HW
* adds new entries after this loop -- the QP we're worried
* about is already in RESET, so the new entries won't come
* from our QP and therefore don't need to be checked.
*/
for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
break;
/*
* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
if ((be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) {
if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));
++nfreed;
} else if (nfreed) {
dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
memcpy(dest, cqe, sizeof *cqe);
dest->owner_sr_opcode = owner_bit |
(dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
}
}
if (nfreed) {
cq->mcq.cons_index += nfreed;
/*
* Make sure update of buffer contents is done before
* updating consumer index.
*/
wmb();
mlx4_cq_set_ci(&cq->mcq);
}
}
static void *get_sw_cqe(struct mlx4_cq *cq, int n)
{
struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe);
struct mlx4_cqe *tcqe = cq->cqe_size == 64 ? cqe + 1 : cqe;
return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
!!(n & (cq->ibv_cq.cqe + 1))) ? NULL : tcqe;
}
void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int old_cqe)
{
struct mlx4_cqe *cqe;
int i;
i = cq->cons_index;
cqe = get_cqe(cq, (i & old_cqe));
while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {
cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) |
(((i + 1) & (cq->ibv_cq.cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0);
memcpy(buf + ((i + 1) & cq->ibv_cq.cqe) * MLX4_CQ_ENTRY_SIZE,
cqe, MLX4_CQ_ENTRY_SIZE);
++i;
cqe = get_cqe(cq, (i & old_cqe));
}
++cq->cons_index;
}
static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq)
{
struct mlx4_cqe *cqe, *new_cqe;
int i;
int cqe_size = cq->buf.entry_size;
int cqe_inc = cqe_size == 64 ? 1 : 0;
i = cq->mcq.cons_index;
cqe = get_cqe(cq, i & cq->ibcq.cqe);
cqe += cqe_inc;
while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {
new_cqe = get_cqe_from_buf(&cq->resize_buf->buf,
(i + 1) & cq->resize_buf->cqe);
memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size);
new_cqe += cqe_inc;
new_cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) |
(((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0);
cqe = get_cqe(cq, ++i & cq->ibcq.cqe);
cqe += cqe_inc;
}
++cq->mcq.cons_index;
}
static void __hns_roce_v1_cq_clean(struct hns_roce_cq *cq, uint32_t qpn,
struct hns_roce_srq *srq)
{
int nfreed = 0;
uint32_t prod_index;
uint8_t owner_bit = 0;
struct hns_roce_cqe *cqe, *dest;
struct hns_roce_context *ctx = to_hr_ctx(cq->ibv_cq.context);
for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index);
++prod_index)
if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
break;
while ((int) --prod_index - (int) cq->cons_index >= 0) {
cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
CQE_BYTE_16_LOCAL_QPN_S) & 0xffffff) == qpn) {
++nfreed;
} else if (nfreed) {
dest = get_cqe(cq,
(prod_index + nfreed) & cq->ibv_cq.cqe);
owner_bit = roce_get_bit(dest->cqe_byte_4,
CQE_BYTE_4_OWNER_S);
memcpy(dest, cqe, sizeof(*cqe));
roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
owner_bit);
}
}
if (nfreed) {
cq->cons_index += nfreed;
udma_to_device_barrier();
hns_roce_update_cq_cons_index(ctx, cq);
}
}
