static void build_rdma_write_cmpl(struct t4_sq *sq,
struct fw_ri_rdma_write_cmpl_wr *wcwr,
struct ibv_send_wr *wr, u8 *len16)
{
u32 plen;
int size;
/*
* This code assumes the struct fields preceding the write isgl fit
* in one 64B WR slot. This is because the WQE is built directly in
* the dma queue, and wrapping is only handled by the code buildling
* sgls. IE the "fixed part" of the wr structs must all fit in 64B.
* The WQE build code should probably be redesigned to avoid this
* restriction, but for now just add a static_assert() to catch if
* this WQE struct gets too big.
*/
static_assert(offsetof(struct fw_ri_rdma_write_cmpl_wr, u) <= 64,
"WQE structure too BIG!");
wcwr->stag_sink = htobe32(wr->wr.rdma.rkey);
wcwr->to_sink = htobe64(wr->wr.rdma.remote_addr);
wcwr->stag_inv = 0;
wcwr->r2 = 0;
wcwr->r3 = 0;
/* SEND_INV SGL */
if (wr->next->send_flags & IBV_SEND_INLINE)
build_immd_cmpl(sq, &wcwr->u_cmpl.immd_src, wr->next);
else
build_isgl((__be64 *)sq->queue, (__be64 *)&sq->queue[sq->size],
&wcwr->u_cmpl.isgl_src, wr->next->sg_list, 1, NULL);
/* WRITE SGL */
build_isgl((__be64 *)sq->queue, (__be64 *)&sq->queue[sq->size],
wcwr->u.isgl_src, wr->sg_list, wr->num_sge, &plen);
size = sizeof(*wcwr) + sizeof(struct fw_ri_isgl) +
wr->num_sge * sizeof(struct fw_ri_sge);
wcwr->plen = htobe32(plen);
*len16 = DIV_ROUND_UP(size, 16);
}
static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
struct ibv_send_wr *wr, u8 *len16)
{
u32 plen;
int size;
int ret;
if (wr->num_sge > T4_MAX_SEND_SGE)
return -EINVAL;
if (wr->send_flags & IBV_SEND_SOLICITED)
wqe->send.sendop_pkd = htobe32(
FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE));
else
wqe->send.sendop_pkd = htobe32(
FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND));
wqe->send.stag_inv = 0;
wqe->send.r3 = 0;
wqe->send.r4 = 0;
plen = 0;
if (wr->num_sge) {
if (wr->send_flags & IBV_SEND_INLINE) {
ret = build_immd(sq, wqe->send.u.immd_src, wr,
T4_MAX_SEND_INLINE, &plen);
if (ret)
return ret;
size = sizeof wqe->send + sizeof(struct fw_ri_immd) +
plen;
} else {
ret = build_isgl((__be64 *)sq->queue,
(__be64 *)&sq->queue[sq->size],
wqe->send.u.isgl_src,
wr->sg_list, wr->num_sge, &plen);
if (ret)
return ret;
size = sizeof wqe->send + sizeof(struct fw_ri_isgl) +
wr->num_sge * sizeof (struct fw_ri_sge);
}
} else {
wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD;
wqe->send.u.immd_src[0].r1 = 0;
wqe->send.u.immd_src[0].r2 = 0;
wqe->send.u.immd_src[0].immdlen = 0;
size = sizeof wqe->send + sizeof(struct fw_ri_immd);
plen = 0;
}
*len16 = DIV_ROUND_UP(size, 16);
wqe->send.plen = htobe32(plen);
return 0;
}
static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
struct ibv_send_wr *wr, u8 *len16)
{
u32 plen;
int size;
int ret;
if (wr->num_sge > T4_MAX_SEND_SGE)
return -EINVAL;
if (wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM)
wqe->write.iw_imm_data.ib_imm_data.imm_data32 = wr->imm_data;
else
wqe->write.iw_imm_data.ib_imm_data.imm_data32 = 0;
wqe->write.stag_sink = htobe32(wr->wr.rdma.rkey);
wqe->write.to_sink = htobe64(wr->wr.rdma.remote_addr);
if (wr->num_sge) {
if (wr->send_flags & IBV_SEND_INLINE) {
ret = build_immd(sq, wqe->write.u.immd_src, wr,
T4_MAX_WRITE_INLINE, &plen);
if (ret)
return ret;
size = sizeof wqe->write + sizeof(struct fw_ri_immd) +
plen;
} else {
ret = build_isgl((__be64 *)sq->queue,
(__be64 *)&sq->queue[sq->size],
wqe->write.u.isgl_src,
wr->sg_list, wr->num_sge, &plen);
if (ret)
return ret;
size = sizeof wqe->write + sizeof(struct fw_ri_isgl) +
wr->num_sge * sizeof (struct fw_ri_sge);
}
} else {
wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD;
wqe->write.u.immd_src[0].r1 = 0;
wqe->write.u.immd_src[0].r2 = 0;
wqe->write.u.immd_src[0].immdlen = 0;
size = sizeof wqe->write + sizeof(struct fw_ri_immd);
plen = 0;
}
*len16 = DIV_ROUND_UP(size, 16);
wqe->write.plen = htobe32(plen);
return 0;
}
static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
struct ibv_send_wr *wr, u8 *len16)
{
u32 plen;
int size;
int ret;
if (wr->num_sge > T4_MAX_SEND_SGE)
return -EINVAL;
wqe->write.r2 = 0;
wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey);
wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr);
if (wr->num_sge) {
if (wr->send_flags & IBV_SEND_INLINE) {
ret = build_immd(sq, wqe->write.u.immd_src, wr,
T4_MAX_WRITE_INLINE, &plen);
if (ret)
return ret;
size = sizeof wqe->write + sizeof(struct fw_ri_immd) +
plen;
} else {
ret = build_isgl(wqe->write.u.isgl_src,
wr->sg_list, wr->num_sge, &plen);
if (ret)
return ret;
size = sizeof wqe->write + sizeof(struct fw_ri_isgl) +
wr->num_sge * sizeof (struct fw_ri_sge);
}
} else {
wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD;
wqe->write.u.immd_src[0].r1 = 0;
wqe->write.u.immd_src[0].r2 = 0;
wqe->write.u.immd_src[0].immdlen = 0;
size = sizeof wqe->write + sizeof(struct fw_ri_immd);
plen = 0;
}
*len16 = DIV_ROUND_UP(size, 16);
wqe->write.plen = cpu_to_be32(plen);
return 0;
}
static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
struct ib_send_wr *wr, u8 *len16)
{
u32 plen;
int size;
int ret;
if (wr->num_sge > T4_MAX_SEND_SGE)
return -EINVAL;
switch (wr->opcode) {
case IB_WR_SEND:
if (wr->send_flags & IB_SEND_SOLICITED)
wqe->send.sendop_pkd = cpu_to_be32(
V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE));
else
wqe->send.sendop_pkd = cpu_to_be32(
V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND));
wqe->send.stag_inv = 0;
break;
case IB_WR_SEND_WITH_INV:
if (wr->send_flags & IB_SEND_SOLICITED)
wqe->send.sendop_pkd = cpu_to_be32(
V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE_INV));
else
wqe->send.sendop_pkd = cpu_to_be32(
V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_INV));
wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
break;
default:
return -EINVAL;
}
plen = 0;
if (wr->num_sge) {
if (wr->send_flags & IB_SEND_INLINE) {
ret = build_immd(sq, wqe->send.u.immd_src, wr,
T4_MAX_SEND_INLINE, &plen);
if (ret)
return ret;
size = sizeof wqe->send + sizeof(struct fw_ri_immd) +
plen;
} else {
ret = build_isgl((__be64 *)sq->queue,
(__be64 *)&sq->queue[sq->size],
wqe->send.u.isgl_src,
wr->sg_list, wr->num_sge, &plen);
if (ret)
return ret;
size = sizeof wqe->send + sizeof(struct fw_ri_isgl) +
wr->num_sge * sizeof(struct fw_ri_sge);
}
} else {
wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD;
wqe->send.u.immd_src[0].r1 = 0;
wqe->send.u.immd_src[0].r2 = 0;
wqe->send.u.immd_src[0].immdlen = 0;
size = sizeof wqe->send + sizeof(struct fw_ri_immd);
plen = 0;
}
*len16 = DIV_ROUND_UP(size, 16);
wqe->send.plen = cpu_to_be32(plen);
return 0;
}
