static int
clnt_setup_long_reply(CONN *conn, struct clist **clpp, uint_t length)
{
if (length == 0) {
*clpp = NULL;
return (CLNT_RDMA_SUCCESS);
}
*clpp = clist_alloc();
(*clpp)->rb_longbuf.len = calc_length(length);
(*clpp)->rb_longbuf.type = RDMA_LONG_BUFFER;
if (rdma_buf_alloc(conn, &((*clpp)->rb_longbuf))) {
clist_free(*clpp);
*clpp = NULL;
return (CLNT_RDMA_FAIL);
}
(*clpp)->u.c_daddr3 = (*clpp)->rb_longbuf.addr;
(*clpp)->c_len = (*clpp)->rb_longbuf.len;
(*clpp)->c_next = NULL;
(*clpp)->c_dmemhandle = (*clpp)->rb_longbuf.handle;
if (clist_register(conn, *clpp, CLIST_REG_DST)) {
DTRACE_PROBE(krpc__e__clntrdma__longrep_regbuf);
rdma_buf_free(conn, &((*clpp)->rb_longbuf));
clist_free(*clpp);
return (CLNT_RDMA_FAIL);
}
return (CLNT_RDMA_SUCCESS);
}
/*
* If xp_cl is NULL value, then the RPC payload will NOT carry
* an RDMA READ chunk list, in this case we insert FALSE into
* the XDR stream. Otherwise we use the clist and RDMA register
* the memory and encode the clist into the outbound XDR stream.
*/
static int
clnt_setup_rlist(CONN *conn, XDR *xdrs, XDR *call_xdrp)
{
int status;
struct clist *rclp;
int32_t xdr_flag = XDR_RDMA_RLIST_REG;
XDR_CONTROL(call_xdrp, XDR_RDMA_GET_RLIST, &rclp);
if (rclp != NULL) {
status = clist_register(conn, rclp, CLIST_REG_SOURCE);
if (status != RDMA_SUCCESS) {
return (CLNT_RDMA_FAIL);
}
XDR_CONTROL(call_xdrp, XDR_RDMA_SET_FLAGS, &xdr_flag);
}
(void) xdr_do_clist(xdrs, &rclp);
return (CLNT_RDMA_SUCCESS);
}
/*
* If xp_wcl is NULL value, then the RPC payload will NOT carry
* an RDMA WRITE chunk list, in this case we insert FALSE into
* the XDR stream. Otherwise we use the clist and RDMA register
* the memory and encode the clist into the outbound XDR stream.
*/
static int
clnt_setup_wlist(CONN *conn, XDR *xdrs, XDR *call_xdrp, rdma_buf_t *rndbuf)
{
int status;
struct clist *wlist, *rndcl;
int wlen, rndlen;
int32_t xdr_flag = XDR_RDMA_WLIST_REG;
XDR_CONTROL(call_xdrp, XDR_RDMA_GET_WLIST, &wlist);
if (wlist != NULL) {
/*
* If we are sending a non 4-byte alligned length
* the server will roundup the length to 4-byte
* boundary. In such a case, a trailing chunk is
* added to take any spill over roundup bytes.
*/
wlen = clist_len(wlist);
rndlen = (roundup(wlen, BYTES_PER_XDR_UNIT) - wlen);
if (rndlen) {
rndcl = clist_alloc();
/*
* calc_length() will allocate a PAGESIZE
* buffer below.
*/
rndcl->c_len = calc_length(rndlen);
rndcl->rb_longbuf.type = RDMA_LONG_BUFFER;
rndcl->rb_longbuf.len = rndcl->c_len;
if (rdma_buf_alloc(conn, &rndcl->rb_longbuf)) {
clist_free(rndcl);
return (CLNT_RDMA_FAIL);
}
/* Roundup buffer freed back in caller */
*rndbuf = rndcl->rb_longbuf;
rndcl->u.c_daddr3 = rndcl->rb_longbuf.addr;
rndcl->c_next = NULL;
rndcl->c_dmemhandle = rndcl->rb_longbuf.handle;
wlist->c_next = rndcl;
}
status = clist_register(conn, wlist, CLIST_REG_DST);
if (status != RDMA_SUCCESS) {
rdma_buf_free(conn, rndbuf);
bzero(rndbuf, sizeof (rdma_buf_t));
return (CLNT_RDMA_FAIL);
}
XDR_CONTROL(call_xdrp, XDR_RDMA_SET_FLAGS, &xdr_flag);
}
if (!xdr_encode_wlist(xdrs, wlist)) {
if (rndlen) {
rdma_buf_free(conn, rndbuf);
bzero(rndbuf, sizeof (rdma_buf_t));
}
return (CLNT_RDMA_FAIL);
}
return (CLNT_RDMA_SUCCESS);
}
/*
* DECODE bytes from XDR stream for rdma.
* If the XDR stream contains a read chunk list,
* it will go through xdrrdma_getrdmablk instead.
*/
static bool_t
xdrrdma_getbytes(XDR *xdrs, caddr_t addr, int len)
{
xrdma_private_t *xdrp = (xrdma_private_t *)(xdrs->x_private);
struct clist *cle = *(xdrp->xp_rcl_next);
struct clist *cls = *(xdrp->xp_rcl_next);
struct clist cl;
bool_t retval = TRUE;
uint32_t total_len = len;
uint32_t cur_offset = 0;
uint32_t total_segments = 0;
uint32_t actual_segments = 0;
uint32_t status = RDMA_SUCCESS;
uint32_t alen = 0;
uint32_t xpoff;
while (cle) {
total_segments++;
cle = cle->c_next;
}
cle = *(xdrp->xp_rcl_next);
if (xdrp->xp_off) {
xpoff = xdrp->xp_off;
} else {
xpoff = (xdrp->xp_offp - xdrs->x_base);
}
/*
* If there was a chunk at the current offset, then setup a read
* chunk list which records the destination address and length
* and will RDMA READ the data in later.
*/
if (cle != NULL && cle->c_xdroff == xpoff) {
for (actual_segments = 0;
actual_segments < total_segments; actual_segments++) {
if (total_len <= 0)
break;
if (status != RDMA_SUCCESS)
goto out;
cle->u.c_daddr = (uint64)(uintptr_t)addr + cur_offset;
alen = 0;
if (cle->c_len > total_len) {
alen = cle->c_len;
cle->c_len = total_len;
}
if (!alen)
xdrp->xp_rcl_next = &cle->c_next;
cur_offset += cle->c_len;
total_len -= cle->c_len;
if ((total_segments - actual_segments - 1) == 0 &&
total_len > 0) {
DTRACE_PROBE(
krpc__e__xdrrdma_getbytes_chunktooshort);
retval = FALSE;
}
if ((total_segments - actual_segments - 1) > 0 &&
total_len == 0) {
DTRACE_PROBE2(krpc__e__xdrrdma_getbytes_toobig,
int, total_segments, int, actual_segments);
}
/*
* RDMA READ the chunk data from the remote end.
* First prep the destination buffer by registering
* it, then RDMA READ the chunk data. Since we are
* doing streaming memory, sync the destination
* buffer to CPU and deregister the buffer.
*/
if (xdrp->xp_conn == NULL) {
return (FALSE);
}
cl = *cle;
cl.c_next = NULL;
status = clist_register(xdrp->xp_conn, &cl,
CLIST_REG_DST);
if (status != RDMA_SUCCESS) {
retval = FALSE;
/*
* Deregister the previous chunks
* before return
*/
goto out;
}
cle->c_dmemhandle = cl.c_dmemhandle;
cle->c_dsynchandle = cl.c_dsynchandle;
/*
* Now read the chunk in
*/
if ((total_segments - actual_segments - 1) == 0 ||
total_len == 0) {
status = RDMA_READ(xdrp->xp_conn, &cl, WAIT);
} else {
status = RDMA_READ(xdrp->xp_conn, &cl, NOWAIT);
}
if (status != RDMA_SUCCESS) {
DTRACE_PROBE1(
krpc__i__xdrrdma_getblk_readfailed,
int, status);
retval = FALSE;
}
cle = cle->c_next;
}
/*
* sync the memory for cpu
*/
cl = *cls;
cl.c_next = NULL;
cl.c_len = cur_offset;
if (clist_syncmem(
xdrp->xp_conn, &cl, CLIST_REG_DST) != RDMA_SUCCESS) {
retval = FALSE;
}
out:
/*
* Deregister the chunks
*/
cle = cls;
while (actual_segments != 0) {
cl = *cle;
cl.c_next = NULL;
cl.c_regtype = CLIST_REG_DST;
(void) clist_deregister(xdrp->xp_conn, &cl);
cle = cle->c_next;
actual_segments--;
}
if (alen) {
cle = *(xdrp->xp_rcl_next);
cle->w.c_saddr =
(uint64)(uintptr_t)cle->w.c_saddr + cle->c_len;
cle->c_len = alen - cle->c_len;
}
return (retval);
}
if ((xdrs->x_handy -= len) < 0)
return (FALSE);
bcopy(xdrp->xp_offp, addr, len);
xdrp->xp_offp += len;
if (xdrp->xp_off != 0)
xdrp->xp_off += len;
return (TRUE);
}
bool_t
xdrrdma_send_read_data(XDR *xdrs, uint_t data_len, struct clist *wcl)
{
int status;
xrdma_private_t *xdrp = (xrdma_private_t *)(xdrs->x_private);
struct xdr_ops *xops = xdrrdma_xops();
struct clist *tcl, *wrcl, *cl;
struct clist fcl;
int rndup_present, rnduplen;
rndup_present = 0;
wrcl = NULL;
/* caller is doing a sizeof */
if (xdrs->x_ops != &xdrrdma_ops || xdrs->x_ops == xops)
return (TRUE);
/* copy of the first chunk */
fcl = *wcl;
fcl.c_next = NULL;
/*
* The entire buffer is registered with the first chunk.
* Later chunks will use the same registered memory handle.
*/
status = clist_register(xdrp->xp_conn, &fcl, CLIST_REG_SOURCE);
if (status != RDMA_SUCCESS) {
return (FALSE);
}
wcl->c_regtype = CLIST_REG_SOURCE;
wcl->c_smemhandle = fcl.c_smemhandle;
wcl->c_ssynchandle = fcl.c_ssynchandle;
/*
* Only transfer the read data ignoring any trailing
* roundup chunks. A bit of work, but it saves an
* unnecessary extra RDMA_WRITE containing only
* roundup bytes.
*/
rnduplen = clist_len(wcl) - data_len;
if (rnduplen) {
tcl = wcl->c_next;
/*
* Check if there is a trailing roundup chunk
*/
while (tcl) {
if ((tcl->c_next == NULL) && (tcl->c_len == rnduplen)) {
rndup_present = 1;
break;
}
tcl = tcl->c_next;
}
/*
* Make a copy chunk list skipping the last chunk
*/
if (rndup_present) {
cl = wcl;
tcl = NULL;
while (cl) {
if (tcl == NULL) {
tcl = clist_alloc();
wrcl = tcl;
} else {
tcl->c_next = clist_alloc();
tcl = tcl->c_next;
}
*tcl = *cl;
cl = cl->c_next;
/* last chunk */
if (cl->c_next == NULL)
break;
}
tcl->c_next = NULL;
}
}
if (wrcl == NULL) {
/* No roundup chunks */
wrcl = wcl;
}
/*
* Set the registered memory handles for the
* rest of the chunks same as the first chunk.
*/
tcl = wrcl->c_next;
while (tcl) {
tcl->c_smemhandle = fcl.c_smemhandle;
tcl->c_ssynchandle = fcl.c_ssynchandle;
tcl = tcl->c_next;
}
/*
* Sync the total len beginning from the first chunk.
*/
fcl.c_len = clist_len(wrcl);
status = clist_syncmem(xdrp->xp_conn, &fcl, CLIST_REG_SOURCE);
if (status != RDMA_SUCCESS) {
return (FALSE);
}
status = RDMA_WRITE(xdrp->xp_conn, wrcl, WAIT);
if (rndup_present)
clist_free(wrcl);
if (status != RDMA_SUCCESS) {
return (FALSE);
}
return (TRUE);
}
bool_t
xdrrdma_read_from_client(struct clist *rlist, CONN **conn, uint_t count)
{
struct clist *rdclist;
struct clist cl;
uint_t total_len = 0;
uint32_t status;
bool_t retval = TRUE;
rlist->rb_longbuf.type = RDMA_LONG_BUFFER;
rlist->rb_longbuf.len =
count > RCL_BUF_LEN ? count : RCL_BUF_LEN;
if (rdma_buf_alloc(*conn, &rlist->rb_longbuf)) {
return (FALSE);
}
/*
* The entire buffer is registered with the first chunk.
* Later chunks will use the same registered memory handle.
*/
cl = *rlist;
cl.c_next = NULL;
if (clist_register(*conn, &cl, CLIST_REG_DST) != RDMA_SUCCESS) {
rdma_buf_free(*conn, &rlist->rb_longbuf);
DTRACE_PROBE(
krpc__e__xdrrdma__readfromclient__clist__reg);
return (FALSE);
}
rlist->c_regtype = CLIST_REG_DST;
rlist->c_dmemhandle = cl.c_dmemhandle;
rlist->c_dsynchandle = cl.c_dsynchandle;
for (rdclist = rlist;
rdclist != NULL; rdclist = rdclist->c_next) {
total_len += rdclist->c_len;
#if (defined(OBJ32)||defined(DEBUG32))
rdclist->u.c_daddr3 =
(caddr_t)((char *)rlist->rb_longbuf.addr +
(uint32) rdclist->u.c_daddr3);
#else
rdclist->u.c_daddr3 =
(caddr_t)((char *)rlist->rb_longbuf.addr +
(uint64) rdclist->u.c_daddr);
#endif
cl = (*rdclist);
cl.c_next = NULL;
/*
* Use the same memory handle for all the chunks
*/
cl.c_dmemhandle = rlist->c_dmemhandle;
cl.c_dsynchandle = rlist->c_dsynchandle;
DTRACE_PROBE1(krpc__i__xdrrdma__readfromclient__buflen,
int, rdclist->c_len);
/*
* Now read the chunk in
*/
if (rdclist->c_next == NULL) {
status = RDMA_READ(*conn, &cl, WAIT);
} else {
status = RDMA_READ(*conn, &cl, NOWAIT);
}
if (status != RDMA_SUCCESS) {
DTRACE_PROBE(
krpc__e__xdrrdma__readfromclient__readfailed);
rdma_buf_free(*conn, &rlist->rb_longbuf);
return (FALSE);
}
}
cl = (*rlist);
cl.c_next = NULL;
cl.c_len = total_len;
if (clist_syncmem(*conn, &cl, CLIST_REG_DST) != RDMA_SUCCESS) {
retval = FALSE;
}
return (retval);
}
