/** * svc_rdma_wc_read - Invoked by RDMA provider for each polled Read WC * @cq: completion queue * @wc: completed WR * */ void svc_rdma_wc_read(struct ib_cq *cq, struct ib_wc *wc) { struct svcxprt_rdma *xprt = cq->cq_context; struct ib_cqe *cqe = wc->wr_cqe; struct svc_rdma_op_ctxt *ctxt; svc_rdma_send_wc_common(xprt, wc, "read"); ctxt = container_of(cqe, struct svc_rdma_op_ctxt, cqe); svc_rdma_unmap_dma(ctxt); svc_rdma_put_frmr(xprt, ctxt->frmr); if (test_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags)) { struct svc_rdma_op_ctxt *read_hdr; read_hdr = ctxt->read_hdr; spin_lock(&xprt->sc_rq_dto_lock); list_add_tail(&read_hdr->dto_q, &xprt->sc_read_complete_q); spin_unlock(&xprt->sc_rq_dto_lock); set_bit(XPT_DATA, &xprt->sc_xprt.xpt_flags); svc_xprt_enqueue(&xprt->sc_xprt); } svc_rdma_put_context(ctxt, 0); svc_xprt_put(&xprt->sc_xprt); }
/* * Processs a completion context */ static void process_context(struct svcxprt_rdma *xprt, struct svc_rdma_op_ctxt *ctxt) { svc_rdma_unmap_dma(ctxt); switch (ctxt->wr_op) { case IB_WR_SEND: if (test_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags)) svc_rdma_put_frmr(xprt, ctxt->frmr); svc_rdma_put_context(ctxt, 1); break; case IB_WR_RDMA_WRITE: svc_rdma_put_context(ctxt, 0); break; case IB_WR_RDMA_READ: case IB_WR_RDMA_READ_WITH_INV: if (test_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags)) { struct svc_rdma_op_ctxt *read_hdr = ctxt->read_hdr; BUG_ON(!read_hdr); if (test_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags)) svc_rdma_put_frmr(xprt, ctxt->frmr); spin_lock_bh(&xprt->sc_rq_dto_lock); set_bit(XPT_DATA, &xprt->sc_xprt.xpt_flags); list_add_tail(&read_hdr->dto_q, &xprt->sc_read_complete_q); spin_unlock_bh(&xprt->sc_rq_dto_lock); svc_xprt_enqueue(&xprt->sc_xprt); } svc_rdma_put_context(ctxt, 0); break; default: printk(KERN_ERR "svcrdma: unexpected completion type, " "opcode=%d\n", ctxt->wr_op); break; } }
/* * Process a completion context */ static void process_context(struct svcxprt_rdma *xprt, struct svc_rdma_op_ctxt *ctxt) { struct svc_rdma_op_ctxt *read_hdr; int free_pages = 0; svc_rdma_unmap_dma(ctxt); switch (ctxt->wr_op) { case IB_WR_SEND: free_pages = 1; break; case IB_WR_RDMA_WRITE: break; case IB_WR_RDMA_READ: case IB_WR_RDMA_READ_WITH_INV: svc_rdma_put_frmr(xprt, ctxt->frmr); if (!test_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags)) break; read_hdr = ctxt->read_hdr; svc_rdma_put_context(ctxt, 0); spin_lock_bh(&xprt->sc_rq_dto_lock); set_bit(XPT_DATA, &xprt->sc_xprt.xpt_flags); list_add_tail(&read_hdr->dto_q, &xprt->sc_read_complete_q); spin_unlock_bh(&xprt->sc_rq_dto_lock); svc_xprt_enqueue(&xprt->sc_xprt); return; default: dprintk("svcrdma: unexpected completion opcode=%d\n", ctxt->wr_op); break; } svc_rdma_put_context(ctxt, free_pages); }
static int send_reply(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp, struct page *page, struct rpcrdma_msg *rdma_resp, struct svc_rdma_op_ctxt *ctxt, struct svc_rdma_req_map *vec, int byte_count) { struct ib_send_wr send_wr; struct ib_send_wr inv_wr; int sge_no; int sge_bytes; int page_no; int ret; /* Post a recv buffer to handle another request. */ ret = svc_rdma_post_recv(rdma); if (ret) { printk(KERN_INFO "svcrdma: could not post a receive buffer, err=%d." "Closing transport %p.\n", ret, rdma); set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); svc_rdma_put_frmr(rdma, vec->frmr); svc_rdma_put_context(ctxt, 0); return -ENOTCONN; } /* Prepare the context */ ctxt->pages[0] = page; ctxt->count = 1; ctxt->frmr = vec->frmr; if (vec->frmr) set_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags); else clear_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags); /* Prepare the SGE for the RPCRDMA Header */ ctxt->sge[0].lkey = rdma->sc_dma_lkey; ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp); ctxt->sge[0].addr = ib_dma_map_single(rdma->sc_cm_id->device, page_address(page), ctxt->sge[0].length, DMA_TO_DEVICE); if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr)) goto err; atomic_inc(&rdma->sc_dma_used); ctxt->direction = DMA_TO_DEVICE; /* Determine how many of our SGE are to be transmitted */ for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) { sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count); byte_count -= sge_bytes; if (!vec->frmr) { ctxt->sge[sge_no].addr = ib_dma_map_single(rdma->sc_cm_id->device, vec->sge[sge_no].iov_base, sge_bytes, DMA_TO_DEVICE); if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[sge_no].addr)) goto err; atomic_inc(&rdma->sc_dma_used); ctxt->sge[sge_no].lkey = rdma->sc_dma_lkey; } else { ctxt->sge[sge_no].addr = (unsigned long) vec->sge[sge_no].iov_base; ctxt->sge[sge_no].lkey = vec->frmr->mr->lkey; } ctxt->sge[sge_no].length = sge_bytes; } BUG_ON(byte_count != 0); /* Save all respages in the ctxt and remove them from the * respages array. They are our pages until the I/O * completes. */ for (page_no = 0; page_no < rqstp->rq_resused; page_no++) { ctxt->pages[page_no+1] = rqstp->rq_respages[page_no]; ctxt->count++; rqstp->rq_respages[page_no] = NULL; /* * If there are more pages than SGE, terminate SGE * list so that svc_rdma_unmap_dma doesn't attempt to * unmap garbage. */ if (page_no+1 >= sge_no) ctxt->sge[page_no+1].length = 0; } BUG_ON(sge_no > rdma->sc_max_sge); memset(&send_wr, 0, sizeof send_wr); ctxt->wr_op = IB_WR_SEND; send_wr.wr_id = (unsigned long)ctxt; send_wr.sg_list = ctxt->sge; send_wr.num_sge = sge_no; send_wr.opcode = IB_WR_SEND; send_wr.send_flags = IB_SEND_SIGNALED; if (vec->frmr) { /* Prepare INVALIDATE WR */ memset(&inv_wr, 0, sizeof inv_wr); inv_wr.opcode = IB_WR_LOCAL_INV; inv_wr.send_flags = IB_SEND_SIGNALED; inv_wr.ex.invalidate_rkey = vec->frmr->mr->lkey; send_wr.next = &inv_wr; } ret = svc_rdma_send(rdma, &send_wr); if (ret) goto err; return 0; err: svc_rdma_unmap_dma(ctxt); svc_rdma_put_frmr(rdma, vec->frmr); svc_rdma_put_context(ctxt, 1); return -EIO; }
static int fast_reg_xdr(struct svcxprt_rdma *xprt, struct xdr_buf *xdr, struct svc_rdma_req_map *vec) { int sge_no; u32 sge_bytes; u32 page_bytes; u32 page_off; int page_no = 0; u8 *frva; struct svc_rdma_fastreg_mr *frmr; frmr = svc_rdma_get_frmr(xprt); if (IS_ERR(frmr)) return -ENOMEM; vec->frmr = frmr; /* Skip the RPCRDMA header */ sge_no = 1; /* Map the head. */ frva = (void *)((unsigned long)(xdr->head[0].iov_base) & PAGE_MASK); vec->sge[sge_no].iov_base = xdr->head[0].iov_base; vec->sge[sge_no].iov_len = xdr->head[0].iov_len; vec->count = 2; sge_no++; /* Build the FRMR */ frmr->kva = frva; frmr->direction = DMA_TO_DEVICE; frmr->access_flags = 0; frmr->map_len = PAGE_SIZE; frmr->page_list_len = 1; frmr->page_list->page_list[page_no] = ib_dma_map_single(xprt->sc_cm_id->device, (void *)xdr->head[0].iov_base, PAGE_SIZE, DMA_TO_DEVICE); if (ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[page_no])) goto fatal_err; atomic_inc(&xprt->sc_dma_used); page_off = xdr->page_base; page_bytes = xdr->page_len + page_off; if (!page_bytes) goto encode_tail; /* Map the pages */ vec->sge[sge_no].iov_base = frva + frmr->map_len + page_off; vec->sge[sge_no].iov_len = page_bytes; sge_no++; while (page_bytes) { struct page *page; page = xdr->pages[page_no++]; sge_bytes = min_t(u32, page_bytes, (PAGE_SIZE - page_off)); page_bytes -= sge_bytes; frmr->page_list->page_list[page_no] = ib_dma_map_single(xprt->sc_cm_id->device, page_address(page), PAGE_SIZE, DMA_TO_DEVICE); if (ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[page_no])) goto fatal_err; atomic_inc(&xprt->sc_dma_used); page_off = 0; /* reset for next time through loop */ frmr->map_len += PAGE_SIZE; frmr->page_list_len++; } vec->count++; encode_tail: /* Map tail */ if (0 == xdr->tail[0].iov_len) goto done; vec->count++; vec->sge[sge_no].iov_len = xdr->tail[0].iov_len; if (((unsigned long)xdr->tail[0].iov_base & PAGE_MASK) == ((unsigned long)xdr->head[0].iov_base & PAGE_MASK)) { /* * If head and tail use the same page, we don't need * to map it again. */ vec->sge[sge_no].iov_base = xdr->tail[0].iov_base; } else { void *va; /* Map another page for the tail */ page_off = (unsigned long)xdr->tail[0].iov_base & ~PAGE_MASK; va = (void *)((unsigned long)xdr->tail[0].iov_base & PAGE_MASK); vec->sge[sge_no].iov_base = frva + frmr->map_len + page_off; frmr->page_list->page_list[page_no] = ib_dma_map_single(xprt->sc_cm_id->device, va, PAGE_SIZE, DMA_TO_DEVICE); if (ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[page_no])) goto fatal_err; atomic_inc(&xprt->sc_dma_used); frmr->map_len += PAGE_SIZE; frmr->page_list_len++; } done: if (svc_rdma_fastreg(xprt, frmr)) goto fatal_err; return 0; fatal_err: printk("svcrdma: Error fast registering memory for xprt %p\n", xprt); vec->frmr = NULL; svc_rdma_put_frmr(xprt, frmr); return -EIO; }
/* Map a read-chunk-list to an XDR and fast register the page-list. * * Assumptions: * - chunk[0] position points to pages[0] at an offset of 0 * - pages[] will be made physically contiguous by creating a one-off memory * region using the fastreg verb. * - byte_count is # of bytes in read-chunk-list * - ch_count is # of chunks in read-chunk-list * * Output: * - sge array pointing into pages[] array. * - chunk_sge array specifying sge index and count for each * chunk in the read list */ static int fast_reg_read_chunks(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp, struct svc_rdma_op_ctxt *head, struct rpcrdma_msg *rmsgp, struct svc_rdma_req_map *rpl_map, struct svc_rdma_req_map *chl_map, int ch_count, int byte_count) { int page_no; int ch_no; u32 offset; struct rpcrdma_read_chunk *ch; struct svc_rdma_fastreg_mr *frmr; int ret = 0; frmr = svc_rdma_get_frmr(xprt); if (IS_ERR(frmr)) return -ENOMEM; head->frmr = frmr; head->arg.head[0] = rqstp->rq_arg.head[0]; head->arg.tail[0] = rqstp->rq_arg.tail[0]; head->arg.pages = &head->pages[head->count]; head->hdr_count = head->count; /* save count of hdr pages */ head->arg.page_base = 0; head->arg.page_len = byte_count; head->arg.len = rqstp->rq_arg.len + byte_count; head->arg.buflen = rqstp->rq_arg.buflen + byte_count; /* Fast register the page list */ frmr->kva = page_address(rqstp->rq_arg.pages[0]); frmr->direction = DMA_FROM_DEVICE; frmr->access_flags = (IB_ACCESS_LOCAL_WRITE|IB_ACCESS_REMOTE_WRITE); frmr->map_len = byte_count; frmr->page_list_len = PAGE_ALIGN(byte_count) >> PAGE_SHIFT; for (page_no = 0; page_no < frmr->page_list_len; page_no++) { frmr->page_list->page_list[page_no] = ib_dma_map_page(xprt->sc_cm_id->device, rqstp->rq_arg.pages[page_no], 0, PAGE_SIZE, DMA_FROM_DEVICE); if (ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[page_no])) goto fatal_err; atomic_inc(&xprt->sc_dma_used); head->arg.pages[page_no] = rqstp->rq_arg.pages[page_no]; } head->count += page_no; /* rq_respages points one past arg pages */ rqstp->rq_respages = &rqstp->rq_arg.pages[page_no]; /* Create the reply and chunk maps */ offset = 0; ch = (struct rpcrdma_read_chunk *)&rmsgp->rm_body.rm_chunks[0]; for (ch_no = 0; ch_no < ch_count; ch_no++) { rpl_map->sge[ch_no].iov_base = frmr->kva + offset; rpl_map->sge[ch_no].iov_len = ch->rc_target.rs_length; chl_map->ch[ch_no].count = 1; chl_map->ch[ch_no].start = ch_no; offset += ch->rc_target.rs_length; ch++; } ret = svc_rdma_fastreg(xprt, frmr); if (ret) goto fatal_err; return ch_no; fatal_err: printk("svcrdma: error fast registering xdr for xprt %p", xprt); svc_rdma_put_frmr(xprt, frmr); return -EIO; }
/* Issue an RDMA_READ using an FRMR to map the data sink */ static int rdma_read_chunk_frmr(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp, struct svc_rdma_op_ctxt *head, int *page_no, u32 *page_offset, u32 rs_handle, u32 rs_length, u64 rs_offset, int last) { struct ib_send_wr read_wr; struct ib_send_wr inv_wr; struct ib_send_wr fastreg_wr; u8 key; int pages_needed = PAGE_ALIGN(*page_offset + rs_length) >> PAGE_SHIFT; struct svc_rdma_op_ctxt *ctxt = svc_rdma_get_context(xprt); struct svc_rdma_fastreg_mr *frmr = svc_rdma_get_frmr(xprt); int ret, read, pno; u32 pg_off = *page_offset; u32 pg_no = *page_no; if (IS_ERR(frmr)) return -ENOMEM; ctxt->direction = DMA_FROM_DEVICE; ctxt->frmr = frmr; pages_needed = min_t(int, pages_needed, xprt->sc_frmr_pg_list_len); read = min_t(int, pages_needed << PAGE_SHIFT, rs_length); frmr->kva = page_address(rqstp->rq_arg.pages[pg_no]); frmr->direction = DMA_FROM_DEVICE; frmr->access_flags = (IB_ACCESS_LOCAL_WRITE|IB_ACCESS_REMOTE_WRITE); frmr->map_len = pages_needed << PAGE_SHIFT; frmr->page_list_len = pages_needed; for (pno = 0; pno < pages_needed; pno++) { int len = min_t(int, rs_length, PAGE_SIZE - pg_off); head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no]; head->arg.page_len += len; head->arg.len += len; if (!pg_off) head->count++; rqstp->rq_respages = &rqstp->rq_arg.pages[pg_no+1]; #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,8,0)) rqstp->rq_next_page = rqstp->rq_respages + 1; #endif frmr->page_list->page_list[pno] = ib_dma_map_page(xprt->sc_cm_id->device, head->arg.pages[pg_no], 0, PAGE_SIZE, DMA_FROM_DEVICE); ret = ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[pno]); if (ret) goto err; atomic_inc(&xprt->sc_dma_used); /* adjust offset and wrap to next page if needed */ pg_off += len; if (pg_off == PAGE_SIZE) { pg_off = 0; pg_no++; } rs_length -= len; } if (last && rs_length == 0) set_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags); else clear_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags); /* Bump the key */ key = (u8)(frmr->mr->lkey & 0x000000FF); ib_update_fast_reg_key(frmr->mr, ++key); ctxt->sge[0].addr = (unsigned long)frmr->kva + *page_offset; ctxt->sge[0].lkey = frmr->mr->lkey; ctxt->sge[0].length = read; ctxt->count = 1; ctxt->read_hdr = head; /* Prepare FASTREG WR */ memset(&fastreg_wr, 0, sizeof(fastreg_wr)); fastreg_wr.opcode = IB_WR_FAST_REG_MR; fastreg_wr.send_flags = IB_SEND_SIGNALED; fastreg_wr.wr.fast_reg.iova_start = (unsigned long)frmr->kva; fastreg_wr.wr.fast_reg.page_list = frmr->page_list; fastreg_wr.wr.fast_reg.page_list_len = frmr->page_list_len; fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; fastreg_wr.wr.fast_reg.length = frmr->map_len; fastreg_wr.wr.fast_reg.access_flags = frmr->access_flags; fastreg_wr.wr.fast_reg.rkey = frmr->mr->lkey; fastreg_wr.next = &read_wr; /* Prepare RDMA_READ */ memset(&read_wr, 0, sizeof(read_wr)); read_wr.send_flags = IB_SEND_SIGNALED; read_wr.wr.rdma.rkey = rs_handle; read_wr.wr.rdma.remote_addr = rs_offset; read_wr.sg_list = ctxt->sge; read_wr.num_sge = 1; if (xprt->sc_dev_caps & SVCRDMA_DEVCAP_READ_W_INV) { read_wr.opcode = IB_WR_RDMA_READ_WITH_INV; read_wr.wr_id = (unsigned long)ctxt; read_wr.ex.invalidate_rkey = ctxt->frmr->mr->lkey; } else { read_wr.opcode = IB_WR_RDMA_READ; read_wr.next = &inv_wr; /* Prepare invalidate */ memset(&inv_wr, 0, sizeof(inv_wr)); inv_wr.wr_id = (unsigned long)ctxt; inv_wr.opcode = IB_WR_LOCAL_INV; inv_wr.send_flags = IB_SEND_SIGNALED | IB_SEND_FENCE; inv_wr.ex.invalidate_rkey = frmr->mr->lkey; } ctxt->wr_op = read_wr.opcode; /* Post the chain */ ret = svc_rdma_send(xprt, &fastreg_wr); if (ret) { pr_err("svcrdma: Error %d posting RDMA_READ\n", ret); set_bit(XPT_CLOSE, &xprt->sc_xprt.xpt_flags); goto err; } /* return current location in page array */ *page_no = pg_no; *page_offset = pg_off; ret = read; atomic_inc(&rdma_stat_read); return ret; err: svc_rdma_unmap_dma(ctxt); svc_rdma_put_context(ctxt, 0); svc_rdma_put_frmr(xprt, frmr); return ret; }
/* Issue an RDMA_READ using an FRMR to map the data sink */ int rdma_read_chunk_frmr(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp, struct svc_rdma_op_ctxt *head, int *page_no, u32 *page_offset, u32 rs_handle, u32 rs_length, u64 rs_offset, bool last) { struct ib_rdma_wr read_wr; struct ib_send_wr inv_wr; struct ib_reg_wr reg_wr; u8 key; int nents = PAGE_ALIGN(*page_offset + rs_length) >> PAGE_SHIFT; struct svc_rdma_op_ctxt *ctxt = svc_rdma_get_context(xprt); struct svc_rdma_fastreg_mr *frmr = svc_rdma_get_frmr(xprt); int ret, read, pno, dma_nents, n; u32 pg_off = *page_offset; u32 pg_no = *page_no; if (IS_ERR(frmr)) return -ENOMEM; ctxt->direction = DMA_FROM_DEVICE; ctxt->frmr = frmr; nents = min_t(unsigned int, nents, xprt->sc_frmr_pg_list_len); read = min_t(int, (nents << PAGE_SHIFT) - *page_offset, rs_length); frmr->direction = DMA_FROM_DEVICE; frmr->access_flags = (IB_ACCESS_LOCAL_WRITE|IB_ACCESS_REMOTE_WRITE); frmr->sg_nents = nents; for (pno = 0; pno < nents; pno++) { int len = min_t(int, rs_length, PAGE_SIZE - pg_off); head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no]; head->arg.page_len += len; head->arg.len += len; if (!pg_off) head->count++; sg_set_page(&frmr->sg[pno], rqstp->rq_arg.pages[pg_no], len, pg_off); rqstp->rq_respages = &rqstp->rq_arg.pages[pg_no+1]; rqstp->rq_next_page = rqstp->rq_respages + 1; /* adjust offset and wrap to next page if needed */ pg_off += len; if (pg_off == PAGE_SIZE) { pg_off = 0; pg_no++; } rs_length -= len; } if (last && rs_length == 0) set_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags); else clear_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags); dma_nents = ib_dma_map_sg(xprt->sc_cm_id->device, frmr->sg, frmr->sg_nents, frmr->direction); if (!dma_nents) { pr_err("svcrdma: failed to dma map sg %p\n", frmr->sg); return -ENOMEM; } atomic_inc(&xprt->sc_dma_used); n = ib_map_mr_sg(frmr->mr, frmr->sg, frmr->sg_nents, PAGE_SIZE); if (unlikely(n != frmr->sg_nents)) { pr_err("svcrdma: failed to map mr %p (%d/%d elements)\n", frmr->mr, n, frmr->sg_nents); return n < 0 ? n : -EINVAL; } /* Bump the key */ key = (u8)(frmr->mr->lkey & 0x000000FF); ib_update_fast_reg_key(frmr->mr, ++key); ctxt->sge[0].addr = frmr->mr->iova; ctxt->sge[0].lkey = frmr->mr->lkey; ctxt->sge[0].length = frmr->mr->length; ctxt->count = 1; ctxt->read_hdr = head; /* Prepare REG WR */ reg_wr.wr.opcode = IB_WR_REG_MR; reg_wr.wr.wr_id = 0; reg_wr.wr.send_flags = IB_SEND_SIGNALED; reg_wr.wr.num_sge = 0; reg_wr.mr = frmr->mr; reg_wr.key = frmr->mr->lkey; reg_wr.access = frmr->access_flags; reg_wr.wr.next = &read_wr.wr; /* Prepare RDMA_READ */ memset(&read_wr, 0, sizeof(read_wr)); read_wr.wr.send_flags = IB_SEND_SIGNALED; read_wr.rkey = rs_handle; read_wr.remote_addr = rs_offset; read_wr.wr.sg_list = ctxt->sge; read_wr.wr.num_sge = 1; if (xprt->sc_dev_caps & SVCRDMA_DEVCAP_READ_W_INV) { read_wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV; read_wr.wr.wr_id = (unsigned long)ctxt; read_wr.wr.ex.invalidate_rkey = ctxt->frmr->mr->lkey; } else { read_wr.wr.opcode = IB_WR_RDMA_READ; read_wr.wr.next = &inv_wr; /* Prepare invalidate */ memset(&inv_wr, 0, sizeof(inv_wr)); inv_wr.wr_id = (unsigned long)ctxt; inv_wr.opcode = IB_WR_LOCAL_INV; inv_wr.send_flags = IB_SEND_SIGNALED | IB_SEND_FENCE; inv_wr.ex.invalidate_rkey = frmr->mr->lkey; } ctxt->wr_op = read_wr.wr.opcode; /* Post the chain */ ret = svc_rdma_send(xprt, ®_wr.wr); if (ret) { pr_err("svcrdma: Error %d posting RDMA_READ\n", ret); set_bit(XPT_CLOSE, &xprt->sc_xprt.xpt_flags); goto err; } /* return current location in page array */ *page_no = pg_no; *page_offset = pg_off; ret = read; atomic_inc(&rdma_stat_read); return ret; err: ib_dma_unmap_sg(xprt->sc_cm_id->device, frmr->sg, frmr->sg_nents, frmr->direction); svc_rdma_put_context(ctxt, 0); svc_rdma_put_frmr(xprt, frmr); return ret; }