int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
struct ibv_send_wr **bad_wr)
{
struct mlx4_context *ctx;
struct mlx4_qp *qp = to_mqp(ibqp);
void *wqe;
struct mlx4_wqe_ctrl_seg *ctrl;
int ind;
int nreq;
int inl = 0;
int ret = 0;
int size;
int i;
pthread_spin_lock(&qp->sq.lock);
/* XXX check that state is OK to post send */
ind = qp->sq.head;
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (wq_overflow(&qp->sq, nreq, to_mcq(qp->ibv_qp.send_cq))) {
ret = -1;
*bad_wr = wr;
goto out;
}
if (wr->num_sge > qp->sq.max_gs) {
ret = -1;
*bad_wr = wr;
goto out;
}
if (wr->opcode >= sizeof mlx4_ib_opcode / sizeof mlx4_ib_opcode[0]) {
ret = -1;
*bad_wr = wr;
goto out;
}
ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
ctrl->xrcrb_flags =
(wr->send_flags & IBV_SEND_SIGNALED ?
htonl(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
(wr->send_flags & IBV_SEND_SOLICITED ?
htonl(MLX4_WQE_CTRL_SOLICIT) : 0) |
qp->sq_signal_bits;
if (wr->opcode == IBV_WR_SEND_WITH_IMM ||
wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM)
ctrl->imm = wr->imm_data;
else
ctrl->imm = 0;
wqe += sizeof *ctrl;
size = sizeof *ctrl / 16;
switch (ibqp->qp_type) {
case IBV_QPT_XRC:
ctrl->xrcrb_flags |= htonl(wr->xrc_remote_srq_num << 8);
/* fall thru */
case IBV_QPT_RC:
case IBV_QPT_UC:
switch (wr->opcode) {
case IBV_WR_ATOMIC_CMP_AND_SWP:
case IBV_WR_ATOMIC_FETCH_AND_ADD:
set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
wr->wr.atomic.rkey);
wqe += sizeof (struct mlx4_wqe_raddr_seg);
set_atomic_seg(wqe, wr);
wqe += sizeof (struct mlx4_wqe_atomic_seg);
size += (sizeof (struct mlx4_wqe_raddr_seg) +
sizeof (struct mlx4_wqe_atomic_seg)) / 16;
break;
case IBV_WR_RDMA_READ:
inl = 1;
/* fall through */
case IBV_WR_RDMA_WRITE:
case IBV_WR_RDMA_WRITE_WITH_IMM:
set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
wr->wr.rdma.rkey);
wqe += sizeof (struct mlx4_wqe_raddr_seg);
size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
break;
default:
/* No extra segments required for sends */
break;
}
break;
case IBV_QPT_UD:
set_datagram_seg(wqe, wr);
wqe += sizeof (struct mlx4_wqe_datagram_seg);
size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
if (to_mah(wr->wr.ud.ah)->tagged) {
ctrl->ins_vlan = 1 << 6;
ctrl->vlan_tag = htons(to_mah(wr->wr.ud.ah)->vlan);
}
break;
default:
break;
}
if (wr->send_flags & IBV_SEND_INLINE && wr->num_sge) {
struct mlx4_wqe_inline_seg *seg;
void *addr;
int len, seg_len;
int num_seg;
int off, to_copy;
inl = 0;
seg = wqe;
wqe += sizeof *seg;
off = ((uintptr_t) wqe) & (MLX4_INLINE_ALIGN - 1);
num_seg = 0;
seg_len = 0;
for (i = 0; i < wr->num_sge; ++i) {
addr = (void *) (uintptr_t) wr->sg_list[i].addr;
len = wr->sg_list[i].length;
inl += len;
if (inl > qp->max_inline_data) {
inl = 0;
ret = -1;
*bad_wr = wr;
goto out;
}
while (len >= MLX4_INLINE_ALIGN - off) {
to_copy = MLX4_INLINE_ALIGN - off;
memcpy(wqe, addr, to_copy);
len -= to_copy;
wqe += to_copy;
addr += to_copy;
seg_len += to_copy;
wmb(); /* see comment below */
seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len);
seg_len = 0;
seg = wqe;
wqe += sizeof *seg;
off = sizeof *seg;
++num_seg;
}
memcpy(wqe, addr, len);
wqe += len;
seg_len += len;
off += len;
}
if (seg_len) {
++num_seg;
/*
* Need a barrier here to make sure
* all the data is visible before the
* byte_count field is set. Otherwise
* the HCA prefetcher could grab the
* 64-byte chunk with this inline
* segment and get a valid (!=
* 0xffffffff) byte count but stale
* data, and end up sending the wrong
* data.
*/
wmb();
seg->byte_count = htonl(MLX4_INLINE_SEG | seg_len);
}
size += (inl + num_seg * sizeof * seg + 15) / 16;
} else {
struct mlx4_wqe_data_seg *seg = wqe;
for (i = wr->num_sge - 1; i >= 0 ; --i)
set_data_seg(seg + i, wr->sg_list + i);
size += wr->num_sge * (sizeof *seg / 16);
}
ctrl->fence_size = (wr->send_flags & IBV_SEND_FENCE ?
MLX4_WQE_CTRL_FENCE : 0) | size;
/*
* Make sure descriptor is fully written before
* setting ownership bit (because HW can start
* executing as soon as we do).
*/
wmb();
ctrl->owner_opcode = htonl(mlx4_ib_opcode[wr->opcode]) |
(ind & qp->sq.wqe_cnt ? htonl(1 << 31) : 0);
/*
* We can improve latency by not stamping the last
* send queue WQE until after ringing the doorbell, so
* only stamp here if there are still more WQEs to post.
*/
if (wr->next)
stamp_send_wqe(qp, (ind + qp->sq_spare_wqes) &
(qp->sq.wqe_cnt - 1));
++ind;
}
out:
ctx = to_mctx(ibqp->context);
if (nreq == 1 && inl && size > 1 && size < ctx->bf_buf_size / 16) {
ctrl->owner_opcode |= htonl((qp->sq.head & 0xffff) << 8);
*(uint32_t *) (&ctrl->vlan_tag) |= qp->doorbell_qpn;
/*
* Make sure that descriptor is written to memory
* before writing to BlueFlame page.
*/
wmb();
++qp->sq.head;
pthread_spin_lock(&ctx->bf_lock);
mlx4_bf_copy(ctx->bf_page + ctx->bf_offset, (unsigned long *) ctrl,
align(size * 16, 64));
wc_wmb();
ctx->bf_offset ^= ctx->bf_buf_size;
pthread_spin_unlock(&ctx->bf_lock);
} else if (nreq) {
qp->sq.head += nreq;
/*
* Make sure that descriptors are written before
* doorbell record.
*/
wmb();
*(uint32_t *) (ctx->uar + MLX4_SEND_DOORBELL) = qp->doorbell_qpn;
}
if (nreq)
stamp_send_wqe(qp, (ind + qp->sq_spare_wqes - 1) &
(qp->sq.wqe_cnt - 1));
pthread_spin_unlock(&qp->sq.lock);
return ret;
}
static int hns_roce_u_v1_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
struct ibv_send_wr **bad_wr)
{
unsigned int ind;
void *wqe;
int nreq;
int ps_opcode, i;
int ret = 0;
struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
struct hns_roce_wqe_data_seg *dseg = NULL;
struct hns_roce_qp *qp = to_hr_qp(ibvqp);
struct hns_roce_context *ctx = to_hr_ctx(ibvqp->context);
pthread_spin_lock(&qp->sq.lock);
/* check that state is OK to post send */
ind = qp->sq.head;
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (hns_roce_wq_overflow(&qp->sq, nreq,
to_hr_cq(qp->ibv_qp.send_cq))) {
ret = -1;
*bad_wr = wr;
goto out;
}
if (wr->num_sge > qp->sq.max_gs) {
ret = -1;
*bad_wr = wr;
printf("wr->num_sge(<=%d) = %d, check failed!\r\n",
qp->sq.max_gs, wr->num_sge);
goto out;
}
ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
for (i = 0; i < wr->num_sge; i++)
ctrl->msg_length = htole32(le32toh(ctrl->msg_length) +
wr->sg_list[i].length);
ctrl->flag |= htole32(((wr->send_flags & IBV_SEND_SIGNALED) ?
HNS_ROCE_WQE_CQ_NOTIFY : 0) |
(wr->send_flags & IBV_SEND_SOLICITED ?
HNS_ROCE_WQE_SE : 0) |
((wr->opcode == IBV_WR_SEND_WITH_IMM ||
wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM) ?
HNS_ROCE_WQE_IMM : 0) |
(wr->send_flags & IBV_SEND_FENCE ?
HNS_ROCE_WQE_FENCE : 0));
if (wr->opcode == IBV_WR_SEND_WITH_IMM ||
wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM)
ctrl->imm_data = htole32(be32toh(wr->imm_data));
wqe += sizeof(struct hns_roce_wqe_ctrl_seg);
/* set remote addr segment */
switch (ibvqp->qp_type) {
case IBV_QPT_RC:
switch (wr->opcode) {
case IBV_WR_RDMA_READ:
ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
wr->wr.rdma.rkey);
break;
case IBV_WR_RDMA_WRITE:
case IBV_WR_RDMA_WRITE_WITH_IMM:
ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
wr->wr.rdma.rkey);
break;
case IBV_WR_SEND:
case IBV_WR_SEND_WITH_IMM:
ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
break;
case IBV_WR_ATOMIC_CMP_AND_SWP:
case IBV_WR_ATOMIC_FETCH_AND_ADD:
default:
ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
break;
}
ctrl->flag |= htole32(ps_opcode);
wqe += sizeof(struct hns_roce_wqe_raddr_seg);
break;
case IBV_QPT_UC:
case IBV_QPT_UD:
default:
break;
}
dseg = wqe;
/* Inline */
if (wr->send_flags & IBV_SEND_INLINE && wr->num_sge) {
if (le32toh(ctrl->msg_length) > qp->max_inline_data) {
ret = -1;
*bad_wr = wr;
printf("inline data len(1-32)=%d, send_flags = 0x%x, check failed!\r\n",
wr->send_flags, ctrl->msg_length);
return ret;
}
for (i = 0; i < wr->num_sge; i++) {
memcpy(wqe,
((void *) (uintptr_t) wr->sg_list[i].addr),
wr->sg_list[i].length);
wqe = wqe + wr->sg_list[i].length;
}
ctrl->flag |= htole32(HNS_ROCE_WQE_INLINE);
} else {
/* set sge */
for (i = 0; i < wr->num_sge; i++)
set_data_seg(dseg+i, wr->sg_list + i);
ctrl->flag |=
htole32(wr->num_sge << HNS_ROCE_WQE_SGE_NUM_BIT);
}
ind++;
}
out:
/* Set DB return */
if (likely(nreq)) {
qp->sq.head += nreq;
hns_roce_update_sq_head(ctx, qp->ibv_qp.qp_num,
qp->port_num - 1, qp->sl,
qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
}
pthread_spin_unlock(&qp->sq.lock);
return ret;
}
