static int send_reply(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp, struct page *page, struct rpcrdma_msg *rdma_resp, struct svc_rdma_op_ctxt *ctxt, struct svc_rdma_req_map *vec, int byte_count) { struct ib_send_wr send_wr; struct ib_send_wr inv_wr; int sge_no; int sge_bytes; int page_no; int ret; /* Post a recv buffer to handle another request. */ ret = svc_rdma_post_recv(rdma); if (ret) { printk(KERN_INFO "svcrdma: could not post a receive buffer, err=%d." "Closing transport %p.\n", ret, rdma); set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); svc_rdma_put_frmr(rdma, vec->frmr); svc_rdma_put_context(ctxt, 0); return -ENOTCONN; } /* Prepare the context */ ctxt->pages[0] = page; ctxt->count = 1; ctxt->frmr = vec->frmr; if (vec->frmr) set_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags); else clear_bit(RDMACTXT_F_FAST_UNREG, &ctxt->flags); /* Prepare the SGE for the RPCRDMA Header */ ctxt->sge[0].lkey = rdma->sc_dma_lkey; ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp); ctxt->sge[0].addr = ib_dma_map_single(rdma->sc_cm_id->device, page_address(page), ctxt->sge[0].length, DMA_TO_DEVICE); if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[0].addr)) goto err; atomic_inc(&rdma->sc_dma_used); ctxt->direction = DMA_TO_DEVICE; /* Determine how many of our SGE are to be transmitted */ for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) { sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count); byte_count -= sge_bytes; if (!vec->frmr) { ctxt->sge[sge_no].addr = ib_dma_map_single(rdma->sc_cm_id->device, vec->sge[sge_no].iov_base, sge_bytes, DMA_TO_DEVICE); if (ib_dma_mapping_error(rdma->sc_cm_id->device, ctxt->sge[sge_no].addr)) goto err; atomic_inc(&rdma->sc_dma_used); ctxt->sge[sge_no].lkey = rdma->sc_dma_lkey; } else { ctxt->sge[sge_no].addr = (unsigned long) vec->sge[sge_no].iov_base; ctxt->sge[sge_no].lkey = vec->frmr->mr->lkey; } ctxt->sge[sge_no].length = sge_bytes; } BUG_ON(byte_count != 0); /* Save all respages in the ctxt and remove them from the * respages array. They are our pages until the I/O * completes. */ for (page_no = 0; page_no < rqstp->rq_resused; page_no++) { ctxt->pages[page_no+1] = rqstp->rq_respages[page_no]; ctxt->count++; rqstp->rq_respages[page_no] = NULL; /* * If there are more pages than SGE, terminate SGE * list so that svc_rdma_unmap_dma doesn't attempt to * unmap garbage. */ if (page_no+1 >= sge_no) ctxt->sge[page_no+1].length = 0; } BUG_ON(sge_no > rdma->sc_max_sge); memset(&send_wr, 0, sizeof send_wr); ctxt->wr_op = IB_WR_SEND; send_wr.wr_id = (unsigned long)ctxt; send_wr.sg_list = ctxt->sge; send_wr.num_sge = sge_no; send_wr.opcode = IB_WR_SEND; send_wr.send_flags = IB_SEND_SIGNALED; if (vec->frmr) { /* Prepare INVALIDATE WR */ memset(&inv_wr, 0, sizeof inv_wr); inv_wr.opcode = IB_WR_LOCAL_INV; inv_wr.send_flags = IB_SEND_SIGNALED; inv_wr.ex.invalidate_rkey = vec->frmr->mr->lkey; send_wr.next = &inv_wr; } ret = svc_rdma_send(rdma, &send_wr); if (ret) goto err; return 0; err: svc_rdma_unmap_dma(ctxt); svc_rdma_put_frmr(rdma, vec->frmr); svc_rdma_put_context(ctxt, 1); return -EIO; }
/* This function prepares the portion of the RPCRDMA message to be * sent in the RDMA_SEND. This function is called after data sent via * RDMA has already been transmitted. There are three cases: * - The RPCRDMA header, RPC header, and payload are all sent in a * single RDMA_SEND. This is the "inline" case. * - The RPCRDMA header and some portion of the RPC header and data * are sent via this RDMA_SEND and another portion of the data is * sent via RDMA. * - The RPCRDMA header [NOMSG] is sent in this RDMA_SEND and the RPC * header and data are all transmitted via RDMA. * In all three cases, this function prepares the RPCRDMA header in * sge[0], the 'type' parameter indicates the type to place in the * RPCRDMA header, and the 'byte_count' field indicates how much of * the XDR to include in this RDMA_SEND. */ static int send_reply(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp, struct page *page, struct rpcrdma_msg *rdma_resp, struct svc_rdma_op_ctxt *ctxt, struct svc_rdma_req_map *vec, int byte_count) { struct ib_send_wr send_wr; int sge_no; int sge_bytes; int page_no; int ret; /* Post a recv buffer to handle another request. */ ret = svc_rdma_post_recv(rdma); if (ret) { printk(KERN_INFO "svcrdma: could not post a receive buffer, err=%d." "Closing transport %p.\n", ret, rdma); set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); svc_rdma_put_context(ctxt, 0); return -ENOTCONN; } /* Prepare the context */ ctxt->pages[0] = page; ctxt->count = 1; /* Prepare the SGE for the RPCRDMA Header */ atomic_inc(&rdma->sc_dma_used); ctxt->sge[0].addr = ib_dma_map_page(rdma->sc_cm_id->device, page, 0, PAGE_SIZE, DMA_TO_DEVICE); ctxt->direction = DMA_TO_DEVICE; ctxt->sge[0].length = svc_rdma_xdr_get_reply_hdr_len(rdma_resp); ctxt->sge[0].lkey = rdma->sc_phys_mr->lkey; /* Determine how many of our SGE are to be transmitted */ for (sge_no = 1; byte_count && sge_no < vec->count; sge_no++) { sge_bytes = min_t(size_t, vec->sge[sge_no].iov_len, byte_count); byte_count -= sge_bytes; atomic_inc(&rdma->sc_dma_used); ctxt->sge[sge_no].addr = ib_dma_map_single(rdma->sc_cm_id->device, vec->sge[sge_no].iov_base, sge_bytes, DMA_TO_DEVICE); ctxt->sge[sge_no].length = sge_bytes; ctxt->sge[sge_no].lkey = rdma->sc_phys_mr->lkey; } BUG_ON(byte_count != 0); /* Save all respages in the ctxt and remove them from the * respages array. They are our pages until the I/O * completes. */ for (page_no = 0; page_no < rqstp->rq_resused; page_no++) { ctxt->pages[page_no+1] = rqstp->rq_respages[page_no]; ctxt->count++; rqstp->rq_respages[page_no] = NULL; /* If there are more pages than SGE, terminate SGE list */ if (page_no+1 >= sge_no) ctxt->sge[page_no+1].length = 0; } BUG_ON(sge_no > rdma->sc_max_sge); memset(&send_wr, 0, sizeof send_wr); ctxt->wr_op = IB_WR_SEND; send_wr.wr_id = (unsigned long)ctxt; send_wr.sg_list = ctxt->sge; send_wr.num_sge = sge_no; send_wr.opcode = IB_WR_SEND; send_wr.send_flags = IB_SEND_SIGNALED; ret = svc_rdma_send(rdma, &send_wr); if (ret) svc_rdma_put_context(ctxt, 1); return ret; }