static struct mbuf *
finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp)
{
struct icl_pdu *ip = &icp->ip;
uint8_t ulp_submode, padding;
struct mbuf *m, *last;
struct iscsi_bhs *bhs;
/*
* Fix up the data segment mbuf first.
*/
m = ip->ip_data_mbuf;
ulp_submode = icc->ulp_submode;
if (m) {
last = m_last(m);
/*
* Round up the data segment to a 4B boundary. Pad with 0 if
* necessary. There will definitely be room in the mbuf.
*/
padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len;
if (padding) {
bzero(mtod(last, uint8_t *) + last->m_len, padding);
last->m_len += padding;
}
} else {
/*
* Append address and data, and optionally, control (ancillary) data to the
* receive queue of a socket. If present, m0 must include a packet header
* with total length. Returns 0 if insufficient mbufs. Does not validate space
* on the receiving sockbuf.
*/
int
sbappendaddr_nospacecheck_locked(struct sockbuf *sb, const struct sockaddr *asa,
struct mbuf *m0, struct mbuf *control)
{
struct mbuf *ctrl_last;
SOCKBUF_LOCK_ASSERT(sb);
ctrl_last = (control == NULL) ? NULL : m_last(control);
return (sbappendaddr_locked_internal(sb, asa, m0, control, ctrl_last));
}
/*
* The procedure xdrmbuf_create initializes a stream descriptor for a
* mbuf.
*/
void
xdrmbuf_create(XDR *xdrs, struct mbuf *m, enum xdr_op op)
{
KASSERT(m != NULL, ("xdrmbuf_create with NULL mbuf chain"));
xdrs->x_op = op;
xdrs->x_ops = &xdrmbuf_ops;
xdrs->x_base = (char *) m;
if (op == XDR_ENCODE) {
m = m_last(m);
xdrs->x_private = m;
xdrs->x_handy = m->m_len;
} else {
xdrs->x_private = m;
xdrs->x_handy = 0;
}
}
void
xdrmbuf_append(XDR *xdrs, struct mbuf *madd)
{
struct mbuf *m;
KASSERT(xdrs->x_ops == &xdrmbuf_ops && xdrs->x_op == XDR_ENCODE,
("xdrmbuf_append: invalid XDR stream"));
if (m_length(madd, NULL) == 0) {
m_freem(madd);
return;
}
m = (struct mbuf *) xdrs->x_private;
m->m_next = madd;
m = m_last(madd);
xdrs->x_private = m;
xdrs->x_handy = m->m_len;
}
void
sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0,
struct mbuf *control)
{
struct mbuf *m, *mlast;
m_clrprotoflags(m0);
m_last(control)->m_next = m0;
SBLASTRECORDCHK(sb);
for (m = control; m->m_next; m = m->m_next)
sballoc(sb, m);
sballoc(sb, m);
mlast = m;
SBLINKRECORD(sb, control);
sb->sb_mbtail = mlast;
SBLASTMBUFCHK(sb);
SBLASTRECORDCHK(sb);
}
static bool_t
svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
struct sockaddr **addrp, struct mbuf **mp)
{
struct cf_conn *cd = (struct cf_conn *) xprt->xp_p1;
struct uio uio;
struct mbuf *m;
XDR xdrs;
int error, rcvflag;
/*
* Serialise access to the socket and our own record parsing
* state.
*/
sx_xlock(&xprt->xp_lock);
for (;;) {
/*
* If we have an mbuf chain in cd->mpending, try to parse a
* record from it, leaving the result in cd->mreq. If we don't
* have a complete record, leave the partial result in
* cd->mreq and try to read more from the socket.
*/
if (cd->mpending) {
/*
* If cd->resid is non-zero, we have part of the
* record already, otherwise we are expecting a record
* marker.
*/
if (!cd->resid) {
/*
* See if there is enough data buffered to
* make up a record marker. Make sure we can
* handle the case where the record marker is
* split across more than one mbuf.
*/
size_t n = 0;
uint32_t header;
m = cd->mpending;
while (n < sizeof(uint32_t) && m) {
n += m->m_len;
m = m->m_next;
}
if (n < sizeof(uint32_t))
goto readmore;
if (cd->mpending->m_len < sizeof(uint32_t))
cd->mpending = m_pullup(cd->mpending,
sizeof(uint32_t));
memcpy(&header, mtod(cd->mpending, uint32_t *),
sizeof(header));
header = ntohl(header);
cd->eor = (header & 0x80000000) != 0;
cd->resid = header & 0x7fffffff;
m_adj(cd->mpending, sizeof(uint32_t));
}
/*
* Start pulling off mbufs from cd->mpending
* until we either have a complete record or
* we run out of data. We use m_split to pull
* data - it will pull as much as possible and
* split the last mbuf if necessary.
*/
while (cd->mpending && cd->resid) {
m = cd->mpending;
if (cd->mpending->m_next
|| cd->mpending->m_len > cd->resid)
cd->mpending = m_split(cd->mpending,
cd->resid, M_WAIT);
else
cd->mpending = NULL;
if (cd->mreq)
m_last(cd->mreq)->m_next = m;
else
cd->mreq = m;
while (m) {
cd->resid -= m->m_len;
m = m->m_next;
}
}
/*
* If cd->resid is zero now, we have managed to
* receive a record fragment from the stream. Check
* for the end-of-record mark to see if we need more.
*/
if (cd->resid == 0) {
if (!cd->eor)
continue;
/*
* Success - we have a complete record in
* cd->mreq.
*/
xdrmbuf_create(&xdrs, cd->mreq, XDR_DECODE);
cd->mreq = NULL;
sx_xunlock(&xprt->xp_lock);
if (! xdr_callmsg(&xdrs, msg)) {
XDR_DESTROY(&xdrs);
return (FALSE);
}
*addrp = NULL;
*mp = xdrmbuf_getall(&xdrs);
XDR_DESTROY(&xdrs);
return (TRUE);
}
}
readmore:
/*
* The socket upcall calls xprt_active() which will eventually
* cause the server to call us here. We attempt to
* read as much as possible from the socket and put
* the result in cd->mpending. If the read fails,
* we have drained both cd->mpending and the socket so
* we can call xprt_inactive().
*/
uio.uio_resid = 1000000000;
uio.uio_td = curthread;
m = NULL;
rcvflag = MSG_DONTWAIT;
CURVNET_SET(xprt->xp_socket->so_vnet);
error = soreceive(xprt->xp_socket, NULL, &uio, &m, NULL,
&rcvflag);
CURVNET_RESTORE();
if (error == EWOULDBLOCK) {
/*
* We must re-test for readability after
* taking the lock to protect us in the case
* where a new packet arrives on the socket
* after our call to soreceive fails with
* EWOULDBLOCK. The pool lock protects us from
* racing the upcall after our soreadable()
* call returns false.
*/
mtx_lock(&xprt->xp_pool->sp_lock);
if (!soreadable(xprt->xp_socket))
xprt_inactive_locked(xprt);
mtx_unlock(&xprt->xp_pool->sp_lock);
sx_xunlock(&xprt->xp_lock);
return (FALSE);
}
if (error) {
SOCKBUF_LOCK(&xprt->xp_socket->so_rcv);
if (xprt->xp_upcallset) {
xprt->xp_upcallset = 0;
soupcall_clear(xprt->xp_socket, SO_RCV);
}
SOCKBUF_UNLOCK(&xprt->xp_socket->so_rcv);
xprt_inactive(xprt);
cd->strm_stat = XPRT_DIED;
sx_xunlock(&xprt->xp_lock);
return (FALSE);
}
if (!m) {
/*
* EOF - the other end has closed the socket.
*/
xprt_inactive(xprt);
cd->strm_stat = XPRT_DIED;
sx_xunlock(&xprt->xp_lock);
return (FALSE);
}
if (cd->mpending)
m_last(cd->mpending)->m_next = m;
else
cd->mpending = m;
}
static bool_t
svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
struct sockaddr **addrp, struct mbuf **mp)
{
struct cf_conn *cd = (struct cf_conn *) xprt->xp_p1;
struct uio uio;
struct mbuf *m;
struct socket* so = xprt->xp_socket;
XDR xdrs;
int error, rcvflag;
uint32_t xid_plus_direction[2];
/*
* Serialise access to the socket and our own record parsing
* state.
*/
sx_xlock(&xprt->xp_lock);
for (;;) {
/* If we have no request ready, check pending queue. */
while (cd->mpending &&
(cd->mreq == NULL || cd->resid != 0 || !cd->eor)) {
if (!svc_vc_process_pending(xprt))
break;
}
/* Process and return complete request in cd->mreq. */
if (cd->mreq != NULL && cd->resid == 0 && cd->eor) {
/*
* Now, check for a backchannel reply.
* The XID is in the first uint32_t of the reply
* and the message direction is the second one.
*/
if ((cd->mreq->m_len >= sizeof(xid_plus_direction) ||
m_length(cd->mreq, NULL) >=
sizeof(xid_plus_direction)) &&
xprt->xp_p2 != NULL) {
m_copydata(cd->mreq, 0,
sizeof(xid_plus_direction),
(char *)xid_plus_direction);
xid_plus_direction[0] =
ntohl(xid_plus_direction[0]);
xid_plus_direction[1] =
ntohl(xid_plus_direction[1]);
/* Check message direction. */
if (xid_plus_direction[1] == REPLY) {
clnt_bck_svccall(xprt->xp_p2,
cd->mreq,
xid_plus_direction[0]);
cd->mreq = NULL;
continue;
}
}
xdrmbuf_create(&xdrs, cd->mreq, XDR_DECODE);
cd->mreq = NULL;
/* Check for next request in a pending queue. */
svc_vc_process_pending(xprt);
if (cd->mreq == NULL || cd->resid != 0) {
SOCKBUF_LOCK(&so->so_rcv);
if (!soreadable(so))
xprt_inactive_self(xprt);
SOCKBUF_UNLOCK(&so->so_rcv);
}
sx_xunlock(&xprt->xp_lock);
if (! xdr_callmsg(&xdrs, msg)) {
XDR_DESTROY(&xdrs);
return (FALSE);
}
*addrp = NULL;
*mp = xdrmbuf_getall(&xdrs);
XDR_DESTROY(&xdrs);
return (TRUE);
}
/*
* The socket upcall calls xprt_active() which will eventually
* cause the server to call us here. We attempt to
* read as much as possible from the socket and put
* the result in cd->mpending. If the read fails,
* we have drained both cd->mpending and the socket so
* we can call xprt_inactive().
*/
uio.uio_resid = 1000000000;
uio.uio_td = curthread;
m = NULL;
rcvflag = MSG_DONTWAIT;
error = soreceive(so, NULL, &uio, &m, NULL, &rcvflag);
if (error == EWOULDBLOCK) {
/*
* We must re-test for readability after
* taking the lock to protect us in the case
* where a new packet arrives on the socket
* after our call to soreceive fails with
* EWOULDBLOCK.
*/
SOCKBUF_LOCK(&so->so_rcv);
if (!soreadable(so))
xprt_inactive_self(xprt);
SOCKBUF_UNLOCK(&so->so_rcv);
sx_xunlock(&xprt->xp_lock);
return (FALSE);
}
if (error) {
SOCKBUF_LOCK(&so->so_rcv);
if (xprt->xp_upcallset) {
xprt->xp_upcallset = 0;
soupcall_clear(so, SO_RCV);
}
SOCKBUF_UNLOCK(&so->so_rcv);
xprt_inactive_self(xprt);
cd->strm_stat = XPRT_DIED;
sx_xunlock(&xprt->xp_lock);
return (FALSE);
}
if (!m) {
/*
* EOF - the other end has closed the socket.
*/
xprt_inactive_self(xprt);
cd->strm_stat = XPRT_DIED;
sx_xunlock(&xprt->xp_lock);
return (FALSE);
}
if (cd->mpending)
m_last(cd->mpending)->m_next = m;
else
cd->mpending = m;
}
}
/*
* If we have an mbuf chain in cd->mpending, try to parse a record from it,
* leaving the result in cd->mreq. If we don't have a complete record, leave
* the partial result in cd->mreq and try to read more from the socket.
*/
static int
svc_vc_process_pending(SVCXPRT *xprt)
{
struct cf_conn *cd = (struct cf_conn *) xprt->xp_p1;
struct socket *so = xprt->xp_socket;
struct mbuf *m;
/*
* If cd->resid is non-zero, we have part of the
* record already, otherwise we are expecting a record
* marker.
*/
if (!cd->resid && cd->mpending) {
/*
* See if there is enough data buffered to
* make up a record marker. Make sure we can
* handle the case where the record marker is
* split across more than one mbuf.
*/
size_t n = 0;
uint32_t header;
m = cd->mpending;
while (n < sizeof(uint32_t) && m) {
n += m->m_len;
m = m->m_next;
}
if (n < sizeof(uint32_t)) {
so->so_rcv.sb_lowat = sizeof(uint32_t) - n;
return (FALSE);
}
m_copydata(cd->mpending, 0, sizeof(header),
(char *)&header);
header = ntohl(header);
cd->eor = (header & 0x80000000) != 0;
cd->resid = header & 0x7fffffff;
m_adj(cd->mpending, sizeof(uint32_t));
}
/*
* Start pulling off mbufs from cd->mpending
* until we either have a complete record or
* we run out of data. We use m_split to pull
* data - it will pull as much as possible and
* split the last mbuf if necessary.
*/
while (cd->mpending && cd->resid) {
m = cd->mpending;
if (cd->mpending->m_next
|| cd->mpending->m_len > cd->resid)
cd->mpending = m_split(cd->mpending,
cd->resid, M_WAITOK);
else
cd->mpending = NULL;
if (cd->mreq)
m_last(cd->mreq)->m_next = m;
else
cd->mreq = m;
while (m) {
cd->resid -= m->m_len;
m = m->m_next;
}
}
/*
* Block receive upcalls if we have more data pending,
* otherwise report our need.
*/
if (cd->mpending)
so->so_rcv.sb_lowat = INT_MAX;
else
so->so_rcv.sb_lowat =
imax(1, imin(cd->resid, so->so_rcv.sb_hiwat / 2));
return (TRUE);
}