static int fast_reg_xdr(struct svcxprt_rdma *xprt, struct xdr_buf *xdr, struct svc_rdma_req_map *vec) { int sge_no; u32 sge_bytes; u32 page_bytes; u32 page_off; int page_no = 0; u8 *frva; struct svc_rdma_fastreg_mr *frmr; frmr = svc_rdma_get_frmr(xprt); if (IS_ERR(frmr)) return -ENOMEM; vec->frmr = frmr; /* Skip the RPCRDMA header */ sge_no = 1; /* Map the head. */ frva = (void *)((unsigned long)(xdr->head[0].iov_base) & PAGE_MASK); vec->sge[sge_no].iov_base = xdr->head[0].iov_base; vec->sge[sge_no].iov_len = xdr->head[0].iov_len; vec->count = 2; sge_no++; /* Build the FRMR */ frmr->kva = frva; frmr->direction = DMA_TO_DEVICE; frmr->access_flags = 0; frmr->map_len = PAGE_SIZE; frmr->page_list_len = 1; frmr->page_list->page_list[page_no] = ib_dma_map_single(xprt->sc_cm_id->device, (void *)xdr->head[0].iov_base, PAGE_SIZE, DMA_TO_DEVICE); if (ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[page_no])) goto fatal_err; atomic_inc(&xprt->sc_dma_used); page_off = xdr->page_base; page_bytes = xdr->page_len + page_off; if (!page_bytes) goto encode_tail; /* Map the pages */ vec->sge[sge_no].iov_base = frva + frmr->map_len + page_off; vec->sge[sge_no].iov_len = page_bytes; sge_no++; while (page_bytes) { struct page *page; page = xdr->pages[page_no++]; sge_bytes = min_t(u32, page_bytes, (PAGE_SIZE - page_off)); page_bytes -= sge_bytes; frmr->page_list->page_list[page_no] = ib_dma_map_single(xprt->sc_cm_id->device, page_address(page), PAGE_SIZE, DMA_TO_DEVICE); if (ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[page_no])) goto fatal_err; atomic_inc(&xprt->sc_dma_used); page_off = 0; /* reset for next time through loop */ frmr->map_len += PAGE_SIZE; frmr->page_list_len++; } vec->count++; encode_tail: /* Map tail */ if (0 == xdr->tail[0].iov_len) goto done; vec->count++; vec->sge[sge_no].iov_len = xdr->tail[0].iov_len; if (((unsigned long)xdr->tail[0].iov_base & PAGE_MASK) == ((unsigned long)xdr->head[0].iov_base & PAGE_MASK)) { /* * If head and tail use the same page, we don't need * to map it again. */ vec->sge[sge_no].iov_base = xdr->tail[0].iov_base; } else { void *va; /* Map another page for the tail */ page_off = (unsigned long)xdr->tail[0].iov_base & ~PAGE_MASK; va = (void *)((unsigned long)xdr->tail[0].iov_base & PAGE_MASK); vec->sge[sge_no].iov_base = frva + frmr->map_len + page_off; frmr->page_list->page_list[page_no] = ib_dma_map_single(xprt->sc_cm_id->device, va, PAGE_SIZE, DMA_TO_DEVICE); if (ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[page_no])) goto fatal_err; atomic_inc(&xprt->sc_dma_used); frmr->map_len += PAGE_SIZE; frmr->page_list_len++; } done: if (svc_rdma_fastreg(xprt, frmr)) goto fatal_err; return 0; fatal_err: printk("svcrdma: Error fast registering memory for xprt %p\n", xprt); vec->frmr = NULL; svc_rdma_put_frmr(xprt, frmr); return -EIO; }
/* Map a read-chunk-list to an XDR and fast register the page-list. * * Assumptions: * - chunk[0] position points to pages[0] at an offset of 0 * - pages[] will be made physically contiguous by creating a one-off memory * region using the fastreg verb. * - byte_count is # of bytes in read-chunk-list * - ch_count is # of chunks in read-chunk-list * * Output: * - sge array pointing into pages[] array. * - chunk_sge array specifying sge index and count for each * chunk in the read list */ static int fast_reg_read_chunks(struct svcxprt_rdma *xprt, struct svc_rqst *rqstp, struct svc_rdma_op_ctxt *head, struct rpcrdma_msg *rmsgp, struct svc_rdma_req_map *rpl_map, struct svc_rdma_req_map *chl_map, int ch_count, int byte_count) { int page_no; int ch_no; u32 offset; struct rpcrdma_read_chunk *ch; struct svc_rdma_fastreg_mr *frmr; int ret = 0; frmr = svc_rdma_get_frmr(xprt); if (IS_ERR(frmr)) return -ENOMEM; head->frmr = frmr; head->arg.head[0] = rqstp->rq_arg.head[0]; head->arg.tail[0] = rqstp->rq_arg.tail[0]; head->arg.pages = &head->pages[head->count]; head->hdr_count = head->count; /* save count of hdr pages */ head->arg.page_base = 0; head->arg.page_len = byte_count; head->arg.len = rqstp->rq_arg.len + byte_count; head->arg.buflen = rqstp->rq_arg.buflen + byte_count; /* Fast register the page list */ frmr->kva = page_address(rqstp->rq_arg.pages[0]); frmr->direction = DMA_FROM_DEVICE; frmr->access_flags = (IB_ACCESS_LOCAL_WRITE|IB_ACCESS_REMOTE_WRITE); frmr->map_len = byte_count; frmr->page_list_len = PAGE_ALIGN(byte_count) >> PAGE_SHIFT; for (page_no = 0; page_no < frmr->page_list_len; page_no++) { frmr->page_list->page_list[page_no] = ib_dma_map_page(xprt->sc_cm_id->device, rqstp->rq_arg.pages[page_no], 0, PAGE_SIZE, DMA_FROM_DEVICE); if (ib_dma_mapping_error(xprt->sc_cm_id->device, frmr->page_list->page_list[page_no])) goto fatal_err; atomic_inc(&xprt->sc_dma_used); head->arg.pages[page_no] = rqstp->rq_arg.pages[page_no]; } head->count += page_no; /* rq_respages points one past arg pages */ rqstp->rq_respages = &rqstp->rq_arg.pages[page_no]; /* Create the reply and chunk maps */ offset = 0; ch = (struct rpcrdma_read_chunk *)&rmsgp->rm_body.rm_chunks[0]; for (ch_no = 0; ch_no < ch_count; ch_no++) { rpl_map->sge[ch_no].iov_base = frmr->kva + offset; rpl_map->sge[ch_no].iov_len = ch->rc_target.rs_length; chl_map->ch[ch_no].count = 1; chl_map->ch[ch_no].start = ch_no; offset += ch->rc_target.rs_length; ch++; } ret = svc_rdma_fastreg(xprt, frmr); if (ret) goto fatal_err; return ch_no; fatal_err: printk("svcrdma: error fast registering xdr for xprt %p", xprt); svc_rdma_put_frmr(xprt, frmr); return -EIO; }