int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq)
{
struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
int ret = 0;
if (hr_dev->hw->destroy_cq) {
ret = hr_dev->hw->destroy_cq(ib_cq);
} else {
hns_roce_free_cq(hr_dev, hr_cq);
hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
if (ib_cq->uobject) {
ib_umem_release(hr_cq->umem);
if (hr_cq->db_en == 1)
hns_roce_db_unmap_user(
to_hr_ucontext(ib_cq->uobject->context),
&hr_cq->db);
} else {
/* Free the buff of stored cq */
hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf,
ib_cq->cqe);
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)
hns_roce_free_db(hr_dev, &hr_cq->db);
}
kfree(hr_cq);
}
return ret;
}
/**
* hns_roce_u_v1_arm_cq - request completion notification on a CQ
* @ibvcq: The completion queue to request notification for.
* @solicited: If non-zero, a event will be generated only for
* the next solicited CQ entry. If zero, any CQ entry,
* solicited or not, will generate an event
*/
static int hns_roce_u_v1_arm_cq(struct ibv_cq *ibvcq, int solicited)
{
uint32_t ci;
uint32_t solicited_flag;
struct hns_roce_cq_db cq_db;
struct hns_roce_cq *cq = to_hr_cq(ibvcq);
ci = cq->cons_index & ((cq->cq_depth << 1) - 1);
solicited_flag = solicited ? HNS_ROCE_CQ_DB_REQ_SOL :
HNS_ROCE_CQ_DB_REQ_NEXT;
cq_db.u32_4 = 0;
cq_db.u32_8 = 0;
roce_set_bit(cq_db.u32_8, CQ_DB_U32_8_HW_SYNC_S, 1);
roce_set_field(cq_db.u32_8, CQ_DB_U32_8_CMD_M, CQ_DB_U32_8_CMD_S, 3);
roce_set_field(cq_db.u32_8, CQ_DB_U32_8_CMD_MDF_M,
CQ_DB_U32_8_CMD_MDF_S, 1);
roce_set_bit(cq_db.u32_8, CQ_DB_U32_8_NOTIFY_TYPE_S, solicited_flag);
roce_set_field(cq_db.u32_8, CQ_DB_U32_8_CQN_M, CQ_DB_U32_8_CQN_S,
cq->cqn);
roce_set_field(cq_db.u32_4, CQ_DB_U32_4_CONS_IDX_M,
CQ_DB_U32_4_CONS_IDX_S, ci);
hns_roce_write64((uint32_t *)&cq_db, to_hr_ctx(ibvcq->context),
ROCEE_DB_OTHERS_L_0_REG);
return 0;
}
static int hns_roce_u_v1_poll_cq(struct ibv_cq *ibvcq, int ne,
struct ibv_wc *wc)
{
int npolled;
int err = CQ_OK;
struct hns_roce_qp *qp = NULL;
struct hns_roce_cq *cq = to_hr_cq(ibvcq);
struct hns_roce_context *ctx = to_hr_ctx(ibvcq->context);
struct hns_roce_device *dev = to_hr_dev(ibvcq->context->device);
pthread_spin_lock(&cq->lock);
for (npolled = 0; npolled < ne; ++npolled) {
err = hns_roce_v1_poll_one(cq, &qp, wc + npolled);
if (err != CQ_OK)
break;
}
if (npolled) {
if (dev->hw_version == HNS_ROCE_HW_VER1) {
*cq->set_ci_db = (cq->cons_index &
((cq->cq_depth << 1) - 1));
mmio_ordered_writes_hack();
}
hns_roce_update_cq_cons_index(ctx, cq);
}
pthread_spin_unlock(&cq->lock);
return err == CQ_POLL_ERR ? err : npolled;
}
static int hns_roce_u_v1_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
struct ibv_recv_wr **bad_wr)
{
int ret = 0;
int nreq;
int ind;
struct ibv_sge *sg;
struct hns_roce_rc_rq_wqe *rq_wqe;
struct hns_roce_qp *qp = to_hr_qp(ibvqp);
struct hns_roce_context *ctx = to_hr_ctx(ibvqp->context);
pthread_spin_lock(&qp->rq.lock);
/* check that state is OK to post receive */
ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (hns_roce_wq_overflow(&qp->rq, nreq,
to_hr_cq(qp->ibv_qp.recv_cq))) {
ret = -1;
*bad_wr = wr;
goto out;
}
if (wr->num_sge > qp->rq.max_gs) {
ret = -1;
*bad_wr = wr;
goto out;
}
rq_wqe = get_recv_wqe(qp, ind);
if (wr->num_sge > HNS_ROCE_RC_RQ_WQE_MAX_SGE_NUM) {
ret = -1;
*bad_wr = wr;
goto out;
}
if (wr->num_sge == HNS_ROCE_RC_RQ_WQE_MAX_SGE_NUM) {
roce_set_field(rq_wqe->u32_2,
RC_RQ_WQE_NUMBER_OF_DATA_SEG_M,
RC_RQ_WQE_NUMBER_OF_DATA_SEG_S,
HNS_ROCE_RC_RQ_WQE_MAX_SGE_NUM);
sg = wr->sg_list;
rq_wqe->va0 = htole64(sg->addr);
rq_wqe->l_key0 = htole32(sg->lkey);
rq_wqe->length0 = htole32(sg->length);
sg = wr->sg_list + 1;
rq_wqe->va1 = htole64(sg->addr);
rq_wqe->l_key1 = htole32(sg->lkey);
rq_wqe->length1 = htole32(sg->length);
} else if (wr->num_sge == HNS_ROCE_RC_RQ_WQE_MAX_SGE_NUM - 1) {
roce_set_field(rq_wqe->u32_2,
RC_RQ_WQE_NUMBER_OF_DATA_SEG_M,
RC_RQ_WQE_NUMBER_OF_DATA_SEG_S,
HNS_ROCE_RC_RQ_WQE_MAX_SGE_NUM - 1);
sg = wr->sg_list;
rq_wqe->va0 = htole64(sg->addr);
rq_wqe->l_key0 = htole32(sg->lkey);
rq_wqe->length0 = htole32(sg->length);
} else if (wr->num_sge == HNS_ROCE_RC_RQ_WQE_MAX_SGE_NUM - 2) {
roce_set_field(rq_wqe->u32_2,
RC_RQ_WQE_NUMBER_OF_DATA_SEG_M,
RC_RQ_WQE_NUMBER_OF_DATA_SEG_S,
HNS_ROCE_RC_RQ_WQE_MAX_SGE_NUM - 2);
}
qp->rq.wrid[ind] = wr->wr_id;
ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
}
out:
if (nreq) {
qp->rq.head += nreq;
hns_roce_update_rq_head(ctx, qp->ibv_qp.qp_num,
qp->rq.head & ((qp->rq.wqe_cnt << 1) - 1));
}
pthread_spin_unlock(&qp->rq.lock);
return ret;
}
static int hns_roce_u_v1_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
struct ibv_send_wr **bad_wr)
{
unsigned int ind;
void *wqe;
int nreq;
int ps_opcode, i;
int ret = 0;
struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
struct hns_roce_wqe_data_seg *dseg = NULL;
struct hns_roce_qp *qp = to_hr_qp(ibvqp);
struct hns_roce_context *ctx = to_hr_ctx(ibvqp->context);
pthread_spin_lock(&qp->sq.lock);
/* check that state is OK to post send */
ind = qp->sq.head;
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (hns_roce_wq_overflow(&qp->sq, nreq,
to_hr_cq(qp->ibv_qp.send_cq))) {
ret = -1;
*bad_wr = wr;
goto out;
}
if (wr->num_sge > qp->sq.max_gs) {
ret = -1;
*bad_wr = wr;
printf("wr->num_sge(<=%d) = %d, check failed!\r\n",
qp->sq.max_gs, wr->num_sge);
goto out;
}
ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
for (i = 0; i < wr->num_sge; i++)
ctrl->msg_length = htole32(le32toh(ctrl->msg_length) +
wr->sg_list[i].length);
ctrl->flag |= htole32(((wr->send_flags & IBV_SEND_SIGNALED) ?
HNS_ROCE_WQE_CQ_NOTIFY : 0) |
(wr->send_flags & IBV_SEND_SOLICITED ?
HNS_ROCE_WQE_SE : 0) |
((wr->opcode == IBV_WR_SEND_WITH_IMM ||
wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM) ?
HNS_ROCE_WQE_IMM : 0) |
(wr->send_flags & IBV_SEND_FENCE ?
HNS_ROCE_WQE_FENCE : 0));
if (wr->opcode == IBV_WR_SEND_WITH_IMM ||
wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM)
ctrl->imm_data = htole32(be32toh(wr->imm_data));
wqe += sizeof(struct hns_roce_wqe_ctrl_seg);
/* set remote addr segment */
switch (ibvqp->qp_type) {
case IBV_QPT_RC:
switch (wr->opcode) {
case IBV_WR_RDMA_READ:
ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
wr->wr.rdma.rkey);
break;
case IBV_WR_RDMA_WRITE:
case IBV_WR_RDMA_WRITE_WITH_IMM:
ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
wr->wr.rdma.rkey);
break;
case IBV_WR_SEND:
case IBV_WR_SEND_WITH_IMM:
ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
break;
case IBV_WR_ATOMIC_CMP_AND_SWP:
case IBV_WR_ATOMIC_FETCH_AND_ADD:
default:
ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
break;
}
ctrl->flag |= htole32(ps_opcode);
wqe += sizeof(struct hns_roce_wqe_raddr_seg);
break;
case IBV_QPT_UC:
case IBV_QPT_UD:
default:
break;
}
dseg = wqe;
/* Inline */
if (wr->send_flags & IBV_SEND_INLINE && wr->num_sge) {
if (le32toh(ctrl->msg_length) > qp->max_inline_data) {
ret = -1;
*bad_wr = wr;
printf("inline data len(1-32)=%d, send_flags = 0x%x, check failed!\r\n",
wr->send_flags, ctrl->msg_length);
return ret;
}
for (i = 0; i < wr->num_sge; i++) {
memcpy(wqe,
((void *) (uintptr_t) wr->sg_list[i].addr),
wr->sg_list[i].length);
wqe = wqe + wr->sg_list[i].length;
}
ctrl->flag |= htole32(HNS_ROCE_WQE_INLINE);
} else {
/* set sge */
for (i = 0; i < wr->num_sge; i++)
set_data_seg(dseg+i, wr->sg_list + i);
ctrl->flag |=
htole32(wr->num_sge << HNS_ROCE_WQE_SGE_NUM_BIT);
}
ind++;
}
out:
/* Set DB return */
if (likely(nreq)) {
qp->sq.head += nreq;
hns_roce_update_sq_head(ctx, qp->ibv_qp.qp_num,
qp->port_num - 1, qp->sl,
qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
}
pthread_spin_unlock(&qp->sq.lock);
return ret;
}
