/** * rdma_rw_ctx_destroy - release all resources allocated by rdma_rw_ctx_init * @ctx: context to release * @qp: queue pair to operate on * @port_num: port num to which the connection is bound * @sg: scatterlist that was used for the READ/WRITE * @sg_cnt: number of entries in @sg * @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ */ void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, struct scatterlist *sg, u32 sg_cnt, enum dma_data_direction dir) { int i; switch (ctx->type) { case RDMA_RW_MR: for (i = 0; i < ctx->nr_ops; i++) ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr); kfree(ctx->reg); break; case RDMA_RW_MULTI_WR: kfree(ctx->map.wrs); kfree(ctx->map.sges); break; case RDMA_RW_SINGLE_WR: break; default: BUG(); break; } /* P2PDMA contexts do not need to be unmapped */ if (!is_pci_p2pdma_page(sg_page(sg))) ib_dma_unmap_sg(qp->pd->device, sg, sg_cnt, dir); }
/** * rdma_rw_ctx_destroy_signature - release all resources allocated by * rdma_rw_ctx_init_signature * @ctx: context to release * @qp: queue pair to operate on * @port_num: port num to which the connection is bound * @sg: scatterlist that was used for the READ/WRITE * @sg_cnt: number of entries in @sg * @prot_sg: scatterlist that was used for the READ/WRITE of the PI * @prot_sg_cnt: number of entries in @prot_sg * @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ */ void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, struct scatterlist *sg, u32 sg_cnt, struct scatterlist *prot_sg, u32 prot_sg_cnt, enum dma_data_direction dir) { if (WARN_ON_ONCE(ctx->type != RDMA_RW_SIG_MR)) return; ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->data.mr); ib_dma_unmap_sg(qp->pd->device, sg, sg_cnt, dir); if (ctx->sig->prot.mr) { ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->prot.mr); ib_dma_unmap_sg(qp->pd->device, prot_sg, prot_sg_cnt, dir); } ib_mr_pool_put(qp, &qp->sig_mrs, ctx->sig->sig_mr); kfree(ctx->sig); }
static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num, struct rdma_rw_reg_ctx *reg, struct scatterlist *sg, u32 sg_cnt, u32 offset) { u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device); u32 nents = min(sg_cnt, pages_per_mr); int count = 0, ret; reg->mr = ib_mr_pool_get(qp, &qp->rdma_mrs); if (!reg->mr) return -EAGAIN; if (reg->mr->need_inval) { reg->inv_wr.opcode = IB_WR_LOCAL_INV; reg->inv_wr.ex.invalidate_rkey = reg->mr->lkey; reg->inv_wr.next = ®->reg_wr.wr; count++; } else { reg->inv_wr.next = NULL; } ret = ib_map_mr_sg(reg->mr, sg, nents, &offset, PAGE_SIZE); if (ret < nents) { ib_mr_pool_put(qp, &qp->rdma_mrs, reg->mr); return -EINVAL; } reg->reg_wr.wr.opcode = IB_WR_REG_MR; reg->reg_wr.mr = reg->mr; reg->reg_wr.access = IB_ACCESS_LOCAL_WRITE; if (rdma_protocol_iwarp(qp->device, port_num)) reg->reg_wr.access |= IB_ACCESS_REMOTE_WRITE; count++; reg->sge.addr = reg->mr->iova; reg->sge.length = reg->mr->length; return count; }
/** * rdma_rw_ctx_signature init - initialize a RW context with signature offload * @ctx: context to initialize * @qp: queue pair to operate on * @port_num: port num to which the connection is bound * @sg: scatterlist to READ/WRITE from/to * @sg_cnt: number of entries in @sg * @prot_sg: scatterlist to READ/WRITE protection information from/to * @prot_sg_cnt: number of entries in @prot_sg * @sig_attrs: signature offloading algorithms * @remote_addr:remote address to read/write (relative to @rkey) * @rkey: remote key to operate on * @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ * * Returns the number of WQEs that will be needed on the workqueue if * successful, or a negative error code. */ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, struct scatterlist *sg, u32 sg_cnt, struct scatterlist *prot_sg, u32 prot_sg_cnt, struct ib_sig_attrs *sig_attrs, u64 remote_addr, u32 rkey, enum dma_data_direction dir) { struct ib_device *dev = qp->pd->device; u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device); struct ib_rdma_wr *rdma_wr; struct ib_send_wr *prev_wr = NULL; int count = 0, ret; if (sg_cnt > pages_per_mr || prot_sg_cnt > pages_per_mr) { pr_err("SG count too large\n"); return -EINVAL; } ret = ib_dma_map_sg(dev, sg, sg_cnt, dir); if (!ret) return -ENOMEM; sg_cnt = ret; ret = ib_dma_map_sg(dev, prot_sg, prot_sg_cnt, dir); if (!ret) { ret = -ENOMEM; goto out_unmap_sg; } prot_sg_cnt = ret; ctx->type = RDMA_RW_SIG_MR; ctx->nr_ops = 1; ctx->sig = kcalloc(1, sizeof(*ctx->sig), GFP_KERNEL); if (!ctx->sig) { ret = -ENOMEM; goto out_unmap_prot_sg; } ret = rdma_rw_init_one_mr(qp, port_num, &ctx->sig->data, sg, sg_cnt, 0); if (ret < 0) goto out_free_ctx; count += ret; prev_wr = &ctx->sig->data.reg_wr.wr; if (prot_sg_cnt) { ret = rdma_rw_init_one_mr(qp, port_num, &ctx->sig->prot, prot_sg, prot_sg_cnt, 0); if (ret < 0) goto out_destroy_data_mr; count += ret; if (ctx->sig->prot.inv_wr.next) prev_wr->next = &ctx->sig->prot.inv_wr; else prev_wr->next = &ctx->sig->prot.reg_wr.wr; prev_wr = &ctx->sig->prot.reg_wr.wr; } else { ctx->sig->prot.mr = NULL; } ctx->sig->sig_mr = ib_mr_pool_get(qp, &qp->sig_mrs); if (!ctx->sig->sig_mr) { ret = -EAGAIN; goto out_destroy_prot_mr; } if (ctx->sig->sig_mr->need_inval) { memset(&ctx->sig->sig_inv_wr, 0, sizeof(ctx->sig->sig_inv_wr)); ctx->sig->sig_inv_wr.opcode = IB_WR_LOCAL_INV; ctx->sig->sig_inv_wr.ex.invalidate_rkey = ctx->sig->sig_mr->rkey; prev_wr->next = &ctx->sig->sig_inv_wr; prev_wr = &ctx->sig->sig_inv_wr; } ctx->sig->sig_wr.wr.opcode = IB_WR_REG_SIG_MR; ctx->sig->sig_wr.wr.wr_cqe = NULL; ctx->sig->sig_wr.wr.sg_list = &ctx->sig->data.sge; ctx->sig->sig_wr.wr.num_sge = 1; ctx->sig->sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE; ctx->sig->sig_wr.sig_attrs = sig_attrs; ctx->sig->sig_wr.sig_mr = ctx->sig->sig_mr; if (prot_sg_cnt) ctx->sig->sig_wr.prot = &ctx->sig->prot.sge; prev_wr->next = &ctx->sig->sig_wr.wr; prev_wr = &ctx->sig->sig_wr.wr; count++; ctx->sig->sig_sge.addr = 0; ctx->sig->sig_sge.length = ctx->sig->data.sge.length; if (sig_attrs->wire.sig_type != IB_SIG_TYPE_NONE) ctx->sig->sig_sge.length += ctx->sig->prot.sge.length; rdma_wr = &ctx->sig->data.wr; rdma_wr->wr.sg_list = &ctx->sig->sig_sge; rdma_wr->wr.num_sge = 1; rdma_wr->remote_addr = remote_addr; rdma_wr->rkey = rkey; if (dir == DMA_TO_DEVICE) rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; else rdma_wr->wr.opcode = IB_WR_RDMA_READ; prev_wr->next = &rdma_wr->wr; prev_wr = &rdma_wr->wr; count++; return count; out_destroy_prot_mr: if (prot_sg_cnt) ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->prot.mr); out_destroy_data_mr: ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->data.mr); out_free_ctx: kfree(ctx->sig); out_unmap_prot_sg: ib_dma_unmap_sg(dev, prot_sg, prot_sg_cnt, dir); out_unmap_sg: ib_dma_unmap_sg(dev, sg, sg_cnt, dir); return ret; }
static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num, struct scatterlist *sg, u32 sg_cnt, u32 offset, u64 remote_addr, u32 rkey, enum dma_data_direction dir) { u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device); int i, j, ret = 0, count = 0; ctx->nr_ops = (sg_cnt + pages_per_mr - 1) / pages_per_mr; ctx->reg = kcalloc(ctx->nr_ops, sizeof(*ctx->reg), GFP_KERNEL); if (!ctx->reg) { ret = -ENOMEM; goto out; } for (i = 0; i < ctx->nr_ops; i++) { struct rdma_rw_reg_ctx *prev = i ? &ctx->reg[i - 1] : NULL; struct rdma_rw_reg_ctx *reg = &ctx->reg[i]; u32 nents = min(sg_cnt, pages_per_mr); ret = rdma_rw_init_one_mr(qp, port_num, reg, sg, sg_cnt, offset); if (ret < 0) goto out_free; count += ret; if (prev) { if (reg->mr->need_inval) prev->wr.wr.next = ®->inv_wr; else prev->wr.wr.next = ®->reg_wr.wr; } reg->reg_wr.wr.next = ®->wr.wr; reg->wr.wr.sg_list = ®->sge; reg->wr.wr.num_sge = 1; reg->wr.remote_addr = remote_addr; reg->wr.rkey = rkey; if (dir == DMA_TO_DEVICE) { reg->wr.wr.opcode = IB_WR_RDMA_WRITE; } else if (!rdma_cap_read_inv(qp->device, port_num)) { reg->wr.wr.opcode = IB_WR_RDMA_READ; } else { reg->wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV; reg->wr.wr.ex.invalidate_rkey = reg->mr->lkey; } count++; remote_addr += reg->sge.length; sg_cnt -= nents; for (j = 0; j < nents; j++) sg = sg_next(sg); offset = 0; } ctx->type = RDMA_RW_MR; return count; out_free: while (--i >= 0) ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr); kfree(ctx->reg); out: return ret; }