/*
* Drop a record off the front of a sockbuf
* and move the next record to the front.
*/
void
sbdroprecord(struct sockbuf *sb)
{
struct mbuf *m, *mn;
m = sb->sb_mb;
if (m) {
sb->sb_mb = m->m_nextpkt;
do {
sbfree(sb, m);
mn = m_free(m);
} while ((m = mn) != NULL);
}
SB_EMPTY_FIXUP(sb);
}
/*
* Drop a record off the front of a sockbuf
* and move the next record to the front.
*/
void
sbdroprecord(struct sockbuf *sb)
{
struct mbuf *m, *mn;
m = sb->sb_mb;
if (m) {
sb->sb_mb = m->m_nextpkt;
do {
sbfree(sb, m);
MFREE(m, mn);
m = mn;
} while (m);
}
SB_EMPTY_FIXUP(sb);
postevent(0, sb, EV_RWBYTES);
}
/*
* Drop a record off the front of a sockbuf and move the next record to the
* front.
*/
void
sbdroprecord_locked(struct sockbuf *sb)
{
struct mbuf *m;
SOCKBUF_LOCK_ASSERT(sb);
m = sb->sb_mb;
if (m) {
sb->sb_mb = m->m_nextpkt;
do {
sbfree(sb, m);
m = m_free(m);
} while (m);
}
SB_EMPTY_FIXUP(sb);
}
static int
kttcp_soreceive(struct socket *so, unsigned long long slen,
unsigned long long *done, struct lwp *l, int *flagsp)
{
struct mbuf *m, **mp;
int flags, len, error, offset, moff, type;
long long orig_resid, resid;
const struct protosw *pr;
struct mbuf *nextrecord;
pr = so->so_proto;
mp = NULL;
type = 0;
resid = orig_resid = slen;
if (flagsp)
flags = *flagsp &~ MSG_EOR;
else
flags = 0;
if (flags & MSG_OOB) {
m = m_get(M_WAIT, MT_DATA);
solock(so);
error = (*pr->pr_usrreqs->pr_recvoob)(so, m, flags & MSG_PEEK);
sounlock(so);
if (error)
goto bad;
do {
resid -= min(resid, m->m_len);
m = m_free(m);
} while (resid && error == 0 && m);
bad:
if (m)
m_freem(m);
return (error);
}
if (mp)
*mp = NULL;
solock(so);
restart:
if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0)
return (error);
m = so->so_rcv.sb_mb;
/*
* If we have less data than requested, block awaiting more
* (subject to any timeout) if:
* 1. the current count is less than the low water mark,
* 2. MSG_WAITALL is set, and it is possible to do the entire
* receive operation at once if we block (resid <= hiwat), or
* 3. MSG_DONTWAIT is not set.
* If MSG_WAITALL is set but resid is larger than the receive buffer,
* we have to do the receive in sections, and thus risk returning
* a short count if a timeout or signal occurs after we start.
*/
if (m == NULL || (((flags & MSG_DONTWAIT) == 0 &&
so->so_rcv.sb_cc < resid) &&
(so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
((flags & MSG_WAITALL) && resid <= so->so_rcv.sb_hiwat)) &&
m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) {
#ifdef DIAGNOSTIC
if (m == NULL && so->so_rcv.sb_cc)
panic("receive 1");
#endif
if (so->so_error) {
if (m)
goto dontblock;
error = so->so_error;
if ((flags & MSG_PEEK) == 0)
so->so_error = 0;
goto release;
}
if (so->so_state & SS_CANTRCVMORE) {
if (m)
goto dontblock;
else
goto release;
}
for (; m; m = m->m_next)
if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
m = so->so_rcv.sb_mb;
goto dontblock;
}
if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
(so->so_proto->pr_flags & PR_CONNREQUIRED)) {
error = ENOTCONN;
goto release;
}
if (resid == 0)
goto release;
if ((so->so_state & SS_NBIO) ||
(flags & (MSG_DONTWAIT|MSG_NBIO))) {
error = EWOULDBLOCK;
goto release;
}
sbunlock(&so->so_rcv);
error = sbwait(&so->so_rcv);
if (error) {
sounlock(so);
return (error);
}
goto restart;
}
dontblock:
/*
* On entry here, m points to the first record of the socket buffer.
* While we process the initial mbufs containing address and control
* info, we save a copy of m->m_nextpkt into nextrecord.
*/
#ifdef notyet /* XXXX */
if (uio->uio_lwp)
uio->uio_lwp->l_ru.ru_msgrcv++;
#endif
KASSERT(m == so->so_rcv.sb_mb);
SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 1");
SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 1");
nextrecord = m->m_nextpkt;
if (pr->pr_flags & PR_ADDR) {
#ifdef DIAGNOSTIC
if (m->m_type != MT_SONAME)
panic("receive 1a");
#endif
orig_resid = 0;
if (flags & MSG_PEEK) {
m = m->m_next;
} else {
sbfree(&so->so_rcv, m);
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
}
while (m && m->m_type == MT_CONTROL && error == 0) {
if (flags & MSG_PEEK) {
m = m->m_next;
} else {
sbfree(&so->so_rcv, m);
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
}
/*
* If m is non-NULL, we have some data to read. From now on,
* make sure to keep sb_lastrecord consistent when working on
* the last packet on the chain (nextrecord == NULL) and we
* change m->m_nextpkt.
*/
if (m) {
if ((flags & MSG_PEEK) == 0) {
m->m_nextpkt = nextrecord;
/*
* If nextrecord == NULL (this is a single chain),
* then sb_lastrecord may not be valid here if m
* was changed earlier.
*/
if (nextrecord == NULL) {
KASSERT(so->so_rcv.sb_mb == m);
so->so_rcv.sb_lastrecord = m;
}
}
type = m->m_type;
if (type == MT_OOBDATA)
flags |= MSG_OOB;
} else {
if ((flags & MSG_PEEK) == 0) {
KASSERT(so->so_rcv.sb_mb == m);
so->so_rcv.sb_mb = nextrecord;
SB_EMPTY_FIXUP(&so->so_rcv);
}
}
SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 2");
SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 2");
moff = 0;
offset = 0;
while (m && resid > 0 && error == 0) {
if (m->m_type == MT_OOBDATA) {
if (type != MT_OOBDATA)
break;
} else if (type == MT_OOBDATA)
break;
#ifdef DIAGNOSTIC
else if (m->m_type != MT_DATA && m->m_type != MT_HEADER)
panic("receive 3");
#endif
so->so_state &= ~SS_RCVATMARK;
len = resid;
if (so->so_oobmark && len > so->so_oobmark - offset)
len = so->so_oobmark - offset;
if (len > m->m_len - moff)
len = m->m_len - moff;
/*
* If mp is set, just pass back the mbufs.
* Otherwise copy them out via the uio, then free.
* Sockbuf must be consistent here (points to current mbuf,
* it points to next record) when we drop priority;
* we must note any additions to the sockbuf when we
* block interrupts again.
*/
resid -= len;
if (len == m->m_len - moff) {
if (m->m_flags & M_EOR)
flags |= MSG_EOR;
if (flags & MSG_PEEK) {
m = m->m_next;
moff = 0;
} else {
nextrecord = m->m_nextpkt;
sbfree(&so->so_rcv, m);
if (mp) {
*mp = m;
mp = &m->m_next;
so->so_rcv.sb_mb = m = m->m_next;
*mp = NULL;
} else {
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
/*
* If m != NULL, we also know that
* so->so_rcv.sb_mb != NULL.
*/
KASSERT(so->so_rcv.sb_mb == m);
if (m) {
m->m_nextpkt = nextrecord;
if (nextrecord == NULL)
so->so_rcv.sb_lastrecord = m;
} else {
so->so_rcv.sb_mb = nextrecord;
SB_EMPTY_FIXUP(&so->so_rcv);
}
SBLASTRECORDCHK(&so->so_rcv,
"kttcp_soreceive 3");
SBLASTMBUFCHK(&so->so_rcv,
"kttcp_soreceive 3");
}
} else {
if (flags & MSG_PEEK)
moff += len;
else {
if (mp) {
sounlock(so);
*mp = m_copym(m, 0, len, M_WAIT);
solock(so);
}
m->m_data += len;
m->m_len -= len;
so->so_rcv.sb_cc -= len;
}
}
if (so->so_oobmark) {
if ((flags & MSG_PEEK) == 0) {
so->so_oobmark -= len;
if (so->so_oobmark == 0) {
so->so_state |= SS_RCVATMARK;
break;
}
} else {
offset += len;
if (offset == so->so_oobmark)
break;
}
}
if (flags & MSG_EOR)
break;
/*
* If the MSG_WAITALL flag is set (for non-atomic socket),
* we must not quit until "uio->uio_resid == 0" or an error
* termination. If a signal/timeout occurs, return
* with a short count but without error.
* Keep sockbuf locked against other readers.
*/
while (flags & MSG_WAITALL && m == NULL && resid > 0 &&
!sosendallatonce(so) && !nextrecord) {
if (so->so_error || so->so_state & SS_CANTRCVMORE)
break;
/*
* If we are peeking and the socket receive buffer is
* full, stop since we can't get more data to peek at.
*/
if ((flags & MSG_PEEK) && sbspace(&so->so_rcv) <= 0)
break;
/*
* If we've drained the socket buffer, tell the
* protocol in case it needs to do something to
* get it filled again.
*/
if ((pr->pr_flags & PR_WANTRCVD) && so->so_pcb) {
(*pr->pr_usrreqs->pr_rcvd)(so, flags, l);
}
SBLASTRECORDCHK(&so->so_rcv,
"kttcp_soreceive sbwait 2");
SBLASTMBUFCHK(&so->so_rcv,
"kttcp_soreceive sbwait 2");
error = sbwait(&so->so_rcv);
if (error) {
sbunlock(&so->so_rcv);
sounlock(so);
return (0);
}
if ((m = so->so_rcv.sb_mb) != NULL)
nextrecord = m->m_nextpkt;
}
}
if (m && pr->pr_flags & PR_ATOMIC) {
flags |= MSG_TRUNC;
if ((flags & MSG_PEEK) == 0)
(void) sbdroprecord(&so->so_rcv);
}
if ((flags & MSG_PEEK) == 0) {
if (m == NULL) {
/*
* First part is an SB_EMPTY_FIXUP(). Second part
* makes sure sb_lastrecord is up-to-date if
* there is still data in the socket buffer.
*/
so->so_rcv.sb_mb = nextrecord;
if (so->so_rcv.sb_mb == NULL) {
so->so_rcv.sb_mbtail = NULL;
so->so_rcv.sb_lastrecord = NULL;
} else if (nextrecord->m_nextpkt == NULL)
so->so_rcv.sb_lastrecord = nextrecord;
}
SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 4");
SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 4");
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb) {
(*pr->pr_usrreqs->pr_rcvd)(so, flags, l);
}
}
if (orig_resid == resid && orig_resid &&
(flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) {
sbunlock(&so->so_rcv);
goto restart;
}
if (flagsp)
*flagsp |= flags;
release:
sbunlock(&so->so_rcv);
sounlock(so);
*done = slen - resid;
#if 0
printf("soreceive: error %d slen %llu resid %lld\n", error, slen, resid);
#endif
return (error);
}
void
sdp_post_sends(struct sdp_sock *ssk, int wait)
{
struct mbuf *mb;
int post_count = 0;
struct socket *sk;
int low;
sk = ssk->socket;
if (unlikely(!ssk->id)) {
if (sk->so_snd.sb_sndptr) {
sdp_dbg(ssk->socket,
"Send on socket without cmid ECONNRESET.\n");
sdp_notify(ssk, ECONNRESET);
}
return;
}
again:
if (sdp_tx_ring_slots_left(ssk) < SDP_TX_SIZE / 2)
sdp_xmit_poll(ssk, 1);
if (ssk->recv_request &&
ring_tail(ssk->rx_ring) >= ssk->recv_request_head &&
tx_credits(ssk) >= SDP_MIN_TX_CREDITS &&
sdp_tx_ring_slots_left(ssk)) {
mb = sdp_alloc_mb_chrcvbuf_ack(sk,
ssk->recv_bytes - SDP_HEAD_SIZE, wait);
if (mb == NULL)
goto allocfail;
ssk->recv_request = 0;
sdp_post_send(ssk, mb);
post_count++;
}
if (tx_credits(ssk) <= SDP_MIN_TX_CREDITS &&
sdp_tx_ring_slots_left(ssk) && sk->so_snd.sb_sndptr &&
sdp_nagle_off(ssk, sk->so_snd.sb_sndptr)) {
SDPSTATS_COUNTER_INC(send_miss_no_credits);
}
while (tx_credits(ssk) > SDP_MIN_TX_CREDITS &&
sdp_tx_ring_slots_left(ssk) && (mb = sk->so_snd.sb_sndptr) &&
sdp_nagle_off(ssk, mb)) {
struct mbuf *n;
SOCKBUF_LOCK(&sk->so_snd);
sk->so_snd.sb_sndptr = mb->m_nextpkt;
sk->so_snd.sb_mb = mb->m_nextpkt;
mb->m_nextpkt = NULL;
SB_EMPTY_FIXUP(&sk->so_snd);
for (n = mb; n != NULL; n = n->m_next)
sbfree(&sk->so_snd, n);
SOCKBUF_UNLOCK(&sk->so_snd);
sdp_post_send(ssk, mb);
post_count++;
}
if (credit_update_needed(ssk) && ssk->state >= TCPS_ESTABLISHED &&
ssk->state < TCPS_FIN_WAIT_2) {
mb = sdp_alloc_mb_data(ssk->socket, wait);
if (mb == NULL)
goto allocfail;
sdp_post_send(ssk, mb);
SDPSTATS_COUNTER_INC(post_send_credits);
post_count++;
}
/* send DisConn if needed
* Do not send DisConn if there is only 1 credit. Compliance with CA4-82
* If one credit is available, an implementation shall only send SDP
* messages that provide additional credits and also do not contain ULP
* payload. */
if ((ssk->flags & SDP_NEEDFIN) && !sk->so_snd.sb_sndptr &&
tx_credits(ssk) > 1) {
mb = sdp_alloc_mb_disconnect(sk, wait);
if (mb == NULL)
goto allocfail;
ssk->flags &= ~SDP_NEEDFIN;
sdp_post_send(ssk, mb);
post_count++;
}
low = (sdp_tx_ring_slots_left(ssk) <= SDP_MIN_TX_CREDITS);
if (post_count || low) {
if (low)
sdp_arm_tx_cq(ssk);
if (sdp_xmit_poll(ssk, low))
goto again;
}
return;
allocfail:
ssk->nagle_last_unacked = -1;
callout_reset(&ssk->nagle_timer, 1, sdp_nagle_timeout, ssk);
return;
}