/*
* Now that we are going to post the WRs we can update the lkey and need_inval
* state on the MRs. If we were doing this at init time, we would get double
* or missing invalidations if a context was initialized but not actually
* posted.
*/
static void rdma_rw_update_lkey(struct rdma_rw_reg_ctx *reg, bool need_inval)
{
reg->mr->need_inval = need_inval;
ib_update_fast_reg_key(reg->mr, ib_inc_rkey(reg->mr->lkey));
reg->reg_wr.key = reg->mr->lkey;
reg->sge.lkey = reg->mr->lkey;
}
/**
* rdma_rw_ctx_wrs - return chain of WRs for a RDMA READ or WRITE operation
* @ctx: context to operate on
* @qp: queue pair to operate on
* @port_num: port num to which the connection is bound
* @cqe: completion queue entry for the last WR
* @chain_wr: WR to append to the posted chain
*
* Return the WR chain for the set of RDMA READ/WRITE operations described by
* @ctx, as well as any memory registration operations needed. If @chain_wr
* is non-NULL the WR it points to will be appended to the chain of WRs posted.
* If @chain_wr is not set @cqe must be set so that the caller gets a
* completion notification.
*/
struct ib_send_wr *rdma_rw_ctx_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
u8 port_num, struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
{
struct ib_send_wr *first_wr, *last_wr;
int i;
switch (ctx->type) {
case RDMA_RW_SIG_MR:
rdma_rw_update_lkey(&ctx->sig->data, true);
if (ctx->sig->prot.mr)
rdma_rw_update_lkey(&ctx->sig->prot, true);
ctx->sig->sig_mr->need_inval = true;
ib_update_fast_reg_key(ctx->sig->sig_mr,
ib_inc_rkey(ctx->sig->sig_mr->lkey));
ctx->sig->sig_sge.lkey = ctx->sig->sig_mr->lkey;
if (ctx->sig->data.inv_wr.next)
first_wr = &ctx->sig->data.inv_wr;
else
first_wr = &ctx->sig->data.reg_wr.wr;
last_wr = &ctx->sig->data.wr.wr;
break;
case RDMA_RW_MR:
for (i = 0; i < ctx->nr_ops; i++) {
rdma_rw_update_lkey(&ctx->reg[i],
ctx->reg[i].wr.wr.opcode !=
IB_WR_RDMA_READ_WITH_INV);
}
if (ctx->reg[0].inv_wr.next)
first_wr = &ctx->reg[0].inv_wr;
else
first_wr = &ctx->reg[0].reg_wr.wr;
last_wr = &ctx->reg[ctx->nr_ops - 1].wr.wr;
break;
case RDMA_RW_MULTI_WR:
first_wr = &ctx->map.wrs[0].wr;
last_wr = &ctx->map.wrs[ctx->nr_ops - 1].wr;
break;
case RDMA_RW_SINGLE_WR:
first_wr = &ctx->single.wr.wr;
last_wr = &ctx->single.wr.wr;
break;
default:
BUG();
}
if (chain_wr) {
last_wr->next = chain_wr;
} else {
last_wr->wr_cqe = cqe;
last_wr->send_flags |= IB_SEND_SIGNALED;
}
return first_wr;
}
static int
iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
struct iser_pi_context *pi_ctx,
struct iser_mem_reg *data_reg,
struct iser_mem_reg *prot_reg,
struct iser_mem_reg *sig_reg)
{
struct iser_tx_desc *tx_desc = &iser_task->desc;
struct ib_sig_attrs *sig_attrs = &tx_desc->sig_attrs;
struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;
struct ib_sig_handover_wr *wr;
struct ib_mr *mr = pi_ctx->sig_mr;
int ret;
memset(sig_attrs, 0, sizeof(*sig_attrs));
ret = iser_set_sig_attrs(iser_task->sc, sig_attrs);
if (ret)
goto err;
iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask);
if (pi_ctx->sig_mr_valid)
iser_inv_rkey(iser_tx_next_wr(tx_desc), mr, cqe);
ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
wr = container_of(iser_tx_next_wr(tx_desc), struct ib_sig_handover_wr,
wr);
wr->wr.opcode = IB_WR_REG_SIG_MR;
wr->wr.wr_cqe = cqe;
wr->wr.sg_list = &data_reg->sge;
wr->wr.num_sge = 1;
wr->wr.send_flags = 0;
wr->sig_attrs = sig_attrs;
wr->sig_mr = mr;
if (scsi_prot_sg_count(iser_task->sc))
wr->prot = &prot_reg->sge;
else
wr->prot = NULL;
wr->access_flags = IB_ACCESS_LOCAL_WRITE |
IB_ACCESS_REMOTE_READ |
IB_ACCESS_REMOTE_WRITE;
pi_ctx->sig_mr_valid = 1;
sig_reg->sge.lkey = mr->lkey;
sig_reg->rkey = mr->rkey;
sig_reg->sge.addr = 0;
sig_reg->sge.length = scsi_transfer_length(iser_task->sc);
iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=%u\n",
sig_reg->sge.lkey, sig_reg->rkey, sig_reg->sge.addr,
sig_reg->sge.length);
err:
return ret;
}
static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
struct iser_data_buf *mem,
struct iser_reg_resources *rsc,
struct iser_mem_reg *reg)
{
struct iser_tx_desc *tx_desc = &iser_task->desc;
struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;
struct ib_mr *mr = rsc->mr;
struct ib_reg_wr *wr;
int n;
if (rsc->mr_valid)
iser_inv_rkey(iser_tx_next_wr(tx_desc), mr, cqe);
ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
n = ib_map_mr_sg(mr, mem->sg, mem->size, NULL, SIZE_4K);
if (unlikely(n != mem->size)) {
iser_err("failed to map sg (%d/%d)\n",
n, mem->size);
return n < 0 ? n : -EINVAL;
}
wr = reg_wr(iser_tx_next_wr(tx_desc));
wr->wr.opcode = IB_WR_REG_MR;
wr->wr.wr_cqe = cqe;
wr->wr.send_flags = 0;
wr->wr.num_sge = 0;
wr->mr = mr;
wr->key = mr->rkey;
wr->access = IB_ACCESS_LOCAL_WRITE |
IB_ACCESS_REMOTE_WRITE |
IB_ACCESS_REMOTE_READ;
rsc->mr_valid = 1;
reg->sge.lkey = mr->lkey;
reg->rkey = mr->rkey;
reg->sge.addr = mr->iova;
reg->sge.length = mr->length;
iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=0x%x\n",
reg->sge.lkey, reg->rkey, reg->sge.addr, reg->sge.length);
return 0;
}
static int modinit(void)
{
return ib_inc_rkey(0);
}
