/** * rdma_rw_mr_factor - return number of MRs required for a payload * @device: device handling the connection * @port_num: port num to which the connection is bound * @maxpages: maximum payload pages per rdma_rw_ctx * * Returns the number of MRs the device requires to move @maxpayload * bytes. The returned value is used during transport creation to * compute max_rdma_ctxts and the size of the transport's Send and * Send Completion Queues. */ unsigned int rdma_rw_mr_factor(struct ib_device *device, u8 port_num, unsigned int maxpages) { unsigned int mr_pages; if (rdma_rw_can_use_mr(device, port_num)) mr_pages = rdma_rw_fr_page_list_len(device); else mr_pages = device->attrs.max_sge_rd; return DIV_ROUND_UP(maxpages, mr_pages); }
int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr) { struct ib_device *dev = qp->pd->device; u32 nr_mrs = 0, nr_sig_mrs = 0; int ret = 0; if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) { nr_sig_mrs = attr->cap.max_rdma_ctxs; nr_mrs = attr->cap.max_rdma_ctxs * 2; } else if (rdma_rw_can_use_mr(dev, attr->port_num)) { nr_mrs = attr->cap.max_rdma_ctxs; } if (nr_mrs) { ret = ib_mr_pool_init(qp, &qp->rdma_mrs, nr_mrs, IB_MR_TYPE_MEM_REG, rdma_rw_fr_page_list_len(dev)); if (ret) { pr_err("%s: failed to allocated %d MRs\n", __func__, nr_mrs); return ret; } } if (nr_sig_mrs) { ret = ib_mr_pool_init(qp, &qp->sig_mrs, nr_sig_mrs, IB_MR_TYPE_SIGNATURE, 2); if (ret) { pr_err("%s: failed to allocated %d SIG MRs\n", __func__, nr_mrs); goto out_free_rdma_mrs; } } return 0; out_free_rdma_mrs: ib_mr_pool_destroy(qp, &qp->rdma_mrs); return ret; }
static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num, struct rdma_rw_reg_ctx *reg, struct scatterlist *sg, u32 sg_cnt, u32 offset) { u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device); u32 nents = min(sg_cnt, pages_per_mr); int count = 0, ret; reg->mr = ib_mr_pool_get(qp, &qp->rdma_mrs); if (!reg->mr) return -EAGAIN; if (reg->mr->need_inval) { reg->inv_wr.opcode = IB_WR_LOCAL_INV; reg->inv_wr.ex.invalidate_rkey = reg->mr->lkey; reg->inv_wr.next = ®->reg_wr.wr; count++; } else { reg->inv_wr.next = NULL; } ret = ib_map_mr_sg(reg->mr, sg, nents, &offset, PAGE_SIZE); if (ret < nents) { ib_mr_pool_put(qp, &qp->rdma_mrs, reg->mr); return -EINVAL; } reg->reg_wr.wr.opcode = IB_WR_REG_MR; reg->reg_wr.mr = reg->mr; reg->reg_wr.access = IB_ACCESS_LOCAL_WRITE; if (rdma_protocol_iwarp(qp->device, port_num)) reg->reg_wr.access |= IB_ACCESS_REMOTE_WRITE; count++; reg->sge.addr = reg->mr->iova; reg->sge.length = reg->mr->length; return count; }
/** * rdma_rw_ctx_signature init - initialize a RW context with signature offload * @ctx: context to initialize * @qp: queue pair to operate on * @port_num: port num to which the connection is bound * @sg: scatterlist to READ/WRITE from/to * @sg_cnt: number of entries in @sg * @prot_sg: scatterlist to READ/WRITE protection information from/to * @prot_sg_cnt: number of entries in @prot_sg * @sig_attrs: signature offloading algorithms * @remote_addr:remote address to read/write (relative to @rkey) * @rkey: remote key to operate on * @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ * * Returns the number of WQEs that will be needed on the workqueue if * successful, or a negative error code. */ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, struct scatterlist *sg, u32 sg_cnt, struct scatterlist *prot_sg, u32 prot_sg_cnt, struct ib_sig_attrs *sig_attrs, u64 remote_addr, u32 rkey, enum dma_data_direction dir) { struct ib_device *dev = qp->pd->device; u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device); struct ib_rdma_wr *rdma_wr; struct ib_send_wr *prev_wr = NULL; int count = 0, ret; if (sg_cnt > pages_per_mr || prot_sg_cnt > pages_per_mr) { pr_err("SG count too large\n"); return -EINVAL; } ret = ib_dma_map_sg(dev, sg, sg_cnt, dir); if (!ret) return -ENOMEM; sg_cnt = ret; ret = ib_dma_map_sg(dev, prot_sg, prot_sg_cnt, dir); if (!ret) { ret = -ENOMEM; goto out_unmap_sg; } prot_sg_cnt = ret; ctx->type = RDMA_RW_SIG_MR; ctx->nr_ops = 1; ctx->sig = kcalloc(1, sizeof(*ctx->sig), GFP_KERNEL); if (!ctx->sig) { ret = -ENOMEM; goto out_unmap_prot_sg; } ret = rdma_rw_init_one_mr(qp, port_num, &ctx->sig->data, sg, sg_cnt, 0); if (ret < 0) goto out_free_ctx; count += ret; prev_wr = &ctx->sig->data.reg_wr.wr; if (prot_sg_cnt) { ret = rdma_rw_init_one_mr(qp, port_num, &ctx->sig->prot, prot_sg, prot_sg_cnt, 0); if (ret < 0) goto out_destroy_data_mr; count += ret; if (ctx->sig->prot.inv_wr.next) prev_wr->next = &ctx->sig->prot.inv_wr; else prev_wr->next = &ctx->sig->prot.reg_wr.wr; prev_wr = &ctx->sig->prot.reg_wr.wr; } else { ctx->sig->prot.mr = NULL; } ctx->sig->sig_mr = ib_mr_pool_get(qp, &qp->sig_mrs); if (!ctx->sig->sig_mr) { ret = -EAGAIN; goto out_destroy_prot_mr; } if (ctx->sig->sig_mr->need_inval) { memset(&ctx->sig->sig_inv_wr, 0, sizeof(ctx->sig->sig_inv_wr)); ctx->sig->sig_inv_wr.opcode = IB_WR_LOCAL_INV; ctx->sig->sig_inv_wr.ex.invalidate_rkey = ctx->sig->sig_mr->rkey; prev_wr->next = &ctx->sig->sig_inv_wr; prev_wr = &ctx->sig->sig_inv_wr; } ctx->sig->sig_wr.wr.opcode = IB_WR_REG_SIG_MR; ctx->sig->sig_wr.wr.wr_cqe = NULL; ctx->sig->sig_wr.wr.sg_list = &ctx->sig->data.sge; ctx->sig->sig_wr.wr.num_sge = 1; ctx->sig->sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE; ctx->sig->sig_wr.sig_attrs = sig_attrs; ctx->sig->sig_wr.sig_mr = ctx->sig->sig_mr; if (prot_sg_cnt) ctx->sig->sig_wr.prot = &ctx->sig->prot.sge; prev_wr->next = &ctx->sig->sig_wr.wr; prev_wr = &ctx->sig->sig_wr.wr; count++; ctx->sig->sig_sge.addr = 0; ctx->sig->sig_sge.length = ctx->sig->data.sge.length; if (sig_attrs->wire.sig_type != IB_SIG_TYPE_NONE) ctx->sig->sig_sge.length += ctx->sig->prot.sge.length; rdma_wr = &ctx->sig->data.wr; rdma_wr->wr.sg_list = &ctx->sig->sig_sge; rdma_wr->wr.num_sge = 1; rdma_wr->remote_addr = remote_addr; rdma_wr->rkey = rkey; if (dir == DMA_TO_DEVICE) rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; else rdma_wr->wr.opcode = IB_WR_RDMA_READ; prev_wr->next = &rdma_wr->wr; prev_wr = &rdma_wr->wr; count++; return count; out_destroy_prot_mr: if (prot_sg_cnt) ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->prot.mr); out_destroy_data_mr: ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->data.mr); out_free_ctx: kfree(ctx->sig); out_unmap_prot_sg: ib_dma_unmap_sg(dev, prot_sg, prot_sg_cnt, dir); out_unmap_sg: ib_dma_unmap_sg(dev, sg, sg_cnt, dir); return ret; }
static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, struct scatterlist *sg, u32 sg_cnt, u32 offset, u64 remote_addr, u32 rkey, enum dma_data_direction dir) { u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device); int i, j, ret = 0, count = 0; ctx->nr_ops = (sg_cnt + pages_per_mr - 1) / pages_per_mr; ctx->reg = kcalloc(ctx->nr_ops, sizeof(*ctx->reg), GFP_KERNEL); if (!ctx->reg) { ret = -ENOMEM; goto out; } for (i = 0; i < ctx->nr_ops; i++) { struct rdma_rw_reg_ctx *prev = i ? &ctx->reg[i - 1] : NULL; struct rdma_rw_reg_ctx *reg = &ctx->reg[i]; u32 nents = min(sg_cnt, pages_per_mr); ret = rdma_rw_init_one_mr(qp, port_num, reg, sg, sg_cnt, offset); if (ret < 0) goto out_free; count += ret; if (prev) { if (reg->mr->need_inval) prev->wr.wr.next = ®->inv_wr; else prev->wr.wr.next = ®->reg_wr.wr; } reg->reg_wr.wr.next = ®->wr.wr; reg->wr.wr.sg_list = ®->sge; reg->wr.wr.num_sge = 1; reg->wr.remote_addr = remote_addr; reg->wr.rkey = rkey; if (dir == DMA_TO_DEVICE) { reg->wr.wr.opcode = IB_WR_RDMA_WRITE; } else if (!rdma_cap_read_inv(qp->device, port_num)) { reg->wr.wr.opcode = IB_WR_RDMA_READ; } else { reg->wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV; reg->wr.wr.ex.invalidate_rkey = reg->mr->lkey; } count++; remote_addr += reg->sge.length; sg_cnt -= nents; for (j = 0; j < nents; j++) sg = sg_next(sg); offset = 0; } ctx->type = RDMA_RW_MR; return count; out_free: while (--i >= 0) ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr); kfree(ctx->reg); out: return ret; }