void
t3_cancel_ubuf(struct toepcb *toep, struct sockbuf *rcv)
{
struct ddp_state *p = &toep->tp_ddp_state;
int ubuf_pending = t3_ddp_ubuf_pending(toep);
int err = 0, count = 0;
if (p->ubuf == NULL)
return;
sockbuf_lock_assert(rcv);
p->cancel_ubuf = 1;
while (ubuf_pending && !(rcv->sb_state & SBS_CANTRCVMORE)) {
CTR3(KTR_TOM,
"t3_cancel_ubuf: flags0 0x%x flags1 0x%x get_tcb_count %d",
p->buf_state[0].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
p->buf_state[1].flags & (DDP_BF_NOFLIP | DDP_BF_NOCOPY),
p->get_tcb_count);
if (p->get_tcb_count == 0)
t3_cancel_ddpbuf(toep, p->cur_buf);
else
CTR5(KTR_TOM, "waiting err=%d get_tcb_count=%d timeo=%d rcv=%p SBS_CANTRCVMORE=%d",
err, p->get_tcb_count, rcv->sb_timeo, rcv,
!!(rcv->sb_state & SBS_CANTRCVMORE));
while (p->get_tcb_count && !(rcv->sb_state & SBS_CANTRCVMORE)) {
if (count & 0xfffffff)
CTR5(KTR_TOM, "waiting err=%d get_tcb_count=%d timeo=%d rcv=%p count=%d",
err, p->get_tcb_count, rcv->sb_timeo, rcv, count);
count++;
err = sbwait(rcv);
}
ubuf_pending = t3_ddp_ubuf_pending(toep);
}
p->cancel_ubuf = 0;
p->user_ddp_pending = 0;
}
int
soreceive(struct socket * so,
struct mbuf **aname,
char * data,
int * datalen,
int flags)
{
struct mbuf * m;
int len;
int error = 0;
int offset;
struct protosw * pr = so->so_proto;
struct mbuf * nextrecord;
int moff;
int lflags;
if (aname)
*aname = 0;
if (flags & MSG_OOB)
{
m = m_get (M_WAIT, MT_RXDATA);
if (m == NULL)
return ENOBUFS;
lflags = flags & MSG_PEEK;
so->so_req = PRU_RCVOOB;
error = (*pr->pr_usrreq)(so, m, LONG2MBUF((long)lflags));
if (error == 0)
{
do
{
len = *datalen;
if (len > (int)m->m_len)
len = m->m_len;
MEMCPY(data, mtod(m, char*), len);
data += len;
*datalen = len;
m = m_free(m);
} while (*datalen && (error == 0) && m);
}
if (m)
m_freem(m);
return (error);
}
restart:
sblock (&so->so_rcv);
INET_TRACE (INETM_IO,
("INET:soreceive sbcc %d soerror %d so_state %d *datalen %d\n",
so->so_rcv.sb_cc, so->so_error, so->so_state, *datalen));
/* If no data is ready, see if we should wait or return */
if (so->so_rcv.sb_cc == 0)
{
if (so->so_error)
{
error = so->so_error;
so->so_error = 0;
goto release;
}
if (so->so_state & SS_CANTRCVMORE)
goto release;
if ((so->so_state & SS_ISCONNECTED) == 0 &&
(so->so_proto->pr_flags & PR_CONNREQUIRED))
{
error = ENOTCONN;
goto release;
}
if (*datalen == 0)
goto release;
if ((so->so_state & SS_NBIO) || (flags & MSG_DONTWAIT))
{
error = EWOULDBLOCK;
goto release;
}
sbunlock(&so->so_rcv);
sbwait(&so->so_rcv);
goto restart;
}
m = so->so_rcv.sb_mb;
if (m == 0)
panic("sorecv 1");
nextrecord = m->m_act;
if (pr->pr_flags & PR_ADDR)
{
if (m->m_type != MT_SONAME)
{
dprintf ("sorecv:type %d not nam", m->m_type);
panic("sorecv 2");
}
if (flags & MSG_PEEK)
{
if (aname)
*aname = m_copy (m, 0, m->m_len);
m = m->m_next;
} else
{
sbfree (&so->so_rcv, m);
if (aname)
{
*aname = m;
m = m->m_next;
(*aname)->m_next = 0;
so->so_rcv.sb_mb = m;
} else
{
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
if (m)
m->m_act = nextrecord;
}
}
moff = 0;
offset = 0;
while (m && (*datalen > 0) && (error == 0))
{
if (m->m_type != MT_RXDATA && m->m_type != MT_HEADER)
panic("sorecv 3");
len = *datalen;
so->so_state &= ~SS_RCVATMARK;
if (so->so_oobmark && (len > (int)(so->so_oobmark - offset)))
len = (int)(so->so_oobmark - offset);
if (len > (int)(m->m_len - moff))
len = m->m_len - moff;
INET_TRACE (INETM_IO, ("INET: soreceive, so %lx %d bytes, flags %x\n",
so, len, flags));
/*
* Copy mbufs info user buffer, then free them.
* Sockbuf must be consistent here (points to current mbuf,
* it points to next record) when we drop priority;
* we must note any additions to the sockbuf when we
* block interrupts again.
*/
MEMCPY(data, (mtod(m, char *) + moff), len);
data += len;
*datalen -= len;
if (len == (int)(m->m_len - moff))
{
if (flags & MSG_PEEK)
{
m = m->m_next;
moff = 0;
} else
{
nextrecord = m->m_act;
sbfree(&so->so_rcv, m);
{
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
if (m)
m->m_act = nextrecord;
}
} else
{
if (flags & MSG_PEEK)
moff += len;
else
{
m->m_data += len;
m->m_len -= len;
so->so_rcv.sb_cc -= len;
}
}
if (so->so_oobmark)
{
if ((flags & MSG_PEEK) == 0)
{
so->so_oobmark -= len;
if (so->so_oobmark == 0)
{
so->so_state |= SS_RCVATMARK;
break;
}
} else
offset += len;
}
}
if ((flags & MSG_PEEK) == 0)
{
if (m == 0)
so->so_rcv.sb_mb = nextrecord;
else if (pr->pr_flags & PR_ATOMIC)
(void) sbdroprecord(&so->so_rcv);
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb)
{
so->so_req = PRU_RCVD;
(*pr->pr_usrreq)(so, (struct mbuf *)0,
(struct mbuf *)0);
}
}
release:
sbunlock(&so->so_rcv);
return (error);
}
int
sosend(struct socket *so,
struct mbuf *nam, /* sockaddr, if UDP socket, NULL if TCP */
char *data, /* data to send */
int *data_length, /* IN/OUT length of (remaining) data */
int flags)
{
struct mbuf *head = (struct mbuf *)NULL;
struct mbuf *m;
int space;
int resid;
int len;
int error = 0;
int dontroute;
int first = 1;
resid = *data_length;
/*
* In theory resid should be unsigned.
* However, space must be signed, as it might be less than 0
* if we over-committed, and we must use a signed comparison
* of space and resid. On the other hand, a negative resid
* causes us to loop sending 0-length segments to the protocol.
*/
if (resid < 0)
return (EINVAL);
INET_TRACE (INETM_IO, ("INET:sosend: so %lx resid %d sb_hiwat %d so_state %x\n",
so, resid, so->so_snd.sb_hiwat, so->so_state));
if (sosendallatonce(so) && (resid > (int)so->so_snd.sb_hiwat))
return (EMSGSIZE);
dontroute = (flags & MSG_DONTROUTE) &&
((so->so_options & SO_DONTROUTE) == 0) &&
(so->so_proto->pr_flags & PR_ATOMIC);
#define snderr(errno) { error = errno; goto release; }
restart:
sblock(&so->so_snd);
do
{
if (so->so_error)
{
error = so->so_error;
so->so_error = 0; /* ??? */
goto release;
}
if (so->so_state & SS_CANTSENDMORE)
snderr(EPIPE);
if ((so->so_state & SS_ISCONNECTED) == 0)
{
if (so->so_proto->pr_flags & PR_CONNREQUIRED)
snderr(ENOTCONN);
if (nam == 0)
snderr(EDESTADDRREQ);
}
if (flags & MSG_OOB)
space = 1024;
else
{
space = (int)sbspace(&so->so_snd);
if ((sosendallatonce(so) && (space < resid)) ||
((resid >= CLBYTES) && (space < CLBYTES) &&
(so->so_snd.sb_cc >= CLBYTES) &&
((so->so_state & SS_NBIO) == 0) &&
((flags & MSG_DONTWAIT) == 0)))
{
if ((so->so_state & SS_NBIO) || (flags & MSG_DONTWAIT))
{
if (first)
error = EWOULDBLOCK;
goto release;
}
sbunlock(&so->so_snd);
sbwait(&so->so_snd);
goto restart;
}
}
if ( space <= 0 )
{
/* no space in socket send buffer - see if we can wait */
if ((so->so_state & SS_NBIO) || (flags & MSG_DONTWAIT))
{
if (first) /* report first error */
error = EWOULDBLOCK;
goto release;
}
/* If blocking socket, let someone else run */
sbunlock(&so->so_snd);
sbwait(&so->so_snd);
goto restart;
}
while (space > 0)
{
len = resid;
if ( so->so_type == SOCK_STREAM )
{
m = m_getwithdata(MT_TXDATA, len);
if (!m)
snderr(ENOBUFS);
MEMCPY(m->m_data, data, len);
so->so_snd.sb_flags |= SB_MBCOMP; /* allow compression */
}
else
{
m = m_get (M_WAIT, MT_TXDATA);
m->m_data = data;
}
INET_TRACE (INETM_IO,
("sosend:got %d bytes so %lx mlen %d, off %d mtod %x\n",
len, so, m->m_len, m->m_off, mtod (m, caddr_t)));
*data_length -= len;
resid -= len;
data += len;
m->m_len = len;
if (head == (struct mbuf *)NULL)
head = m;
if (error)
goto release;
if (*data_length <= 0)
break;
}
if (dontroute)
so->so_options |= SO_DONTROUTE;
so->so_req = (flags & MSG_OOB) ? PRU_SENDOOB : PRU_SEND;
error = (*so->so_proto->pr_usrreq)(so, head, nam);
if (dontroute)
so->so_options &= ~SO_DONTROUTE;
head = (struct mbuf *)NULL;
first = 0;
} while ((resid != 0) && (error == 0));
release:
sbunlock(&so->so_snd);
if (head)
m_freem(head);
return error;
}
static int
kttcp_soreceive(struct socket *so, unsigned long long slen,
unsigned long long *done, struct lwp *l, int *flagsp)
{
struct mbuf *m, **mp;
int flags, len, error, offset, moff, type;
long long orig_resid, resid;
const struct protosw *pr;
struct mbuf *nextrecord;
pr = so->so_proto;
mp = NULL;
type = 0;
resid = orig_resid = slen;
if (flagsp)
flags = *flagsp &~ MSG_EOR;
else
flags = 0;
if (flags & MSG_OOB) {
m = m_get(M_WAIT, MT_DATA);
solock(so);
error = (*pr->pr_usrreqs->pr_recvoob)(so, m, flags & MSG_PEEK);
sounlock(so);
if (error)
goto bad;
do {
resid -= min(resid, m->m_len);
m = m_free(m);
} while (resid && error == 0 && m);
bad:
if (m)
m_freem(m);
return (error);
}
if (mp)
*mp = NULL;
solock(so);
restart:
if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0)
return (error);
m = so->so_rcv.sb_mb;
/*
* If we have less data than requested, block awaiting more
* (subject to any timeout) if:
* 1. the current count is less than the low water mark,
* 2. MSG_WAITALL is set, and it is possible to do the entire
* receive operation at once if we block (resid <= hiwat), or
* 3. MSG_DONTWAIT is not set.
* If MSG_WAITALL is set but resid is larger than the receive buffer,
* we have to do the receive in sections, and thus risk returning
* a short count if a timeout or signal occurs after we start.
*/
if (m == NULL || (((flags & MSG_DONTWAIT) == 0 &&
so->so_rcv.sb_cc < resid) &&
(so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
((flags & MSG_WAITALL) && resid <= so->so_rcv.sb_hiwat)) &&
m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) {
#ifdef DIAGNOSTIC
if (m == NULL && so->so_rcv.sb_cc)
panic("receive 1");
#endif
if (so->so_error) {
if (m)
goto dontblock;
error = so->so_error;
if ((flags & MSG_PEEK) == 0)
so->so_error = 0;
goto release;
}
if (so->so_state & SS_CANTRCVMORE) {
if (m)
goto dontblock;
else
goto release;
}
for (; m; m = m->m_next)
if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
m = so->so_rcv.sb_mb;
goto dontblock;
}
if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
(so->so_proto->pr_flags & PR_CONNREQUIRED)) {
error = ENOTCONN;
goto release;
}
if (resid == 0)
goto release;
if ((so->so_state & SS_NBIO) ||
(flags & (MSG_DONTWAIT|MSG_NBIO))) {
error = EWOULDBLOCK;
goto release;
}
sbunlock(&so->so_rcv);
error = sbwait(&so->so_rcv);
if (error) {
sounlock(so);
return (error);
}
goto restart;
}
dontblock:
/*
* On entry here, m points to the first record of the socket buffer.
* While we process the initial mbufs containing address and control
* info, we save a copy of m->m_nextpkt into nextrecord.
*/
#ifdef notyet /* XXXX */
if (uio->uio_lwp)
uio->uio_lwp->l_ru.ru_msgrcv++;
#endif
KASSERT(m == so->so_rcv.sb_mb);
SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 1");
SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 1");
nextrecord = m->m_nextpkt;
if (pr->pr_flags & PR_ADDR) {
#ifdef DIAGNOSTIC
if (m->m_type != MT_SONAME)
panic("receive 1a");
#endif
orig_resid = 0;
if (flags & MSG_PEEK) {
m = m->m_next;
} else {
sbfree(&so->so_rcv, m);
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
}
while (m && m->m_type == MT_CONTROL && error == 0) {
if (flags & MSG_PEEK) {
m = m->m_next;
} else {
sbfree(&so->so_rcv, m);
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
}
/*
* If m is non-NULL, we have some data to read. From now on,
* make sure to keep sb_lastrecord consistent when working on
* the last packet on the chain (nextrecord == NULL) and we
* change m->m_nextpkt.
*/
if (m) {
if ((flags & MSG_PEEK) == 0) {
m->m_nextpkt = nextrecord;
/*
* If nextrecord == NULL (this is a single chain),
* then sb_lastrecord may not be valid here if m
* was changed earlier.
*/
if (nextrecord == NULL) {
KASSERT(so->so_rcv.sb_mb == m);
so->so_rcv.sb_lastrecord = m;
}
}
type = m->m_type;
if (type == MT_OOBDATA)
flags |= MSG_OOB;
} else {
if ((flags & MSG_PEEK) == 0) {
KASSERT(so->so_rcv.sb_mb == m);
so->so_rcv.sb_mb = nextrecord;
SB_EMPTY_FIXUP(&so->so_rcv);
}
}
SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 2");
SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 2");
moff = 0;
offset = 0;
while (m && resid > 0 && error == 0) {
if (m->m_type == MT_OOBDATA) {
if (type != MT_OOBDATA)
break;
} else if (type == MT_OOBDATA)
break;
#ifdef DIAGNOSTIC
else if (m->m_type != MT_DATA && m->m_type != MT_HEADER)
panic("receive 3");
#endif
so->so_state &= ~SS_RCVATMARK;
len = resid;
if (so->so_oobmark && len > so->so_oobmark - offset)
len = so->so_oobmark - offset;
if (len > m->m_len - moff)
len = m->m_len - moff;
/*
* If mp is set, just pass back the mbufs.
* Otherwise copy them out via the uio, then free.
* Sockbuf must be consistent here (points to current mbuf,
* it points to next record) when we drop priority;
* we must note any additions to the sockbuf when we
* block interrupts again.
*/
resid -= len;
if (len == m->m_len - moff) {
if (m->m_flags & M_EOR)
flags |= MSG_EOR;
if (flags & MSG_PEEK) {
m = m->m_next;
moff = 0;
} else {
nextrecord = m->m_nextpkt;
sbfree(&so->so_rcv, m);
if (mp) {
*mp = m;
mp = &m->m_next;
so->so_rcv.sb_mb = m = m->m_next;
*mp = NULL;
} else {
MFREE(m, so->so_rcv.sb_mb);
m = so->so_rcv.sb_mb;
}
/*
* If m != NULL, we also know that
* so->so_rcv.sb_mb != NULL.
*/
KASSERT(so->so_rcv.sb_mb == m);
if (m) {
m->m_nextpkt = nextrecord;
if (nextrecord == NULL)
so->so_rcv.sb_lastrecord = m;
} else {
so->so_rcv.sb_mb = nextrecord;
SB_EMPTY_FIXUP(&so->so_rcv);
}
SBLASTRECORDCHK(&so->so_rcv,
"kttcp_soreceive 3");
SBLASTMBUFCHK(&so->so_rcv,
"kttcp_soreceive 3");
}
} else {
if (flags & MSG_PEEK)
moff += len;
else {
if (mp) {
sounlock(so);
*mp = m_copym(m, 0, len, M_WAIT);
solock(so);
}
m->m_data += len;
m->m_len -= len;
so->so_rcv.sb_cc -= len;
}
}
if (so->so_oobmark) {
if ((flags & MSG_PEEK) == 0) {
so->so_oobmark -= len;
if (so->so_oobmark == 0) {
so->so_state |= SS_RCVATMARK;
break;
}
} else {
offset += len;
if (offset == so->so_oobmark)
break;
}
}
if (flags & MSG_EOR)
break;
/*
* If the MSG_WAITALL flag is set (for non-atomic socket),
* we must not quit until "uio->uio_resid == 0" or an error
* termination. If a signal/timeout occurs, return
* with a short count but without error.
* Keep sockbuf locked against other readers.
*/
while (flags & MSG_WAITALL && m == NULL && resid > 0 &&
!sosendallatonce(so) && !nextrecord) {
if (so->so_error || so->so_state & SS_CANTRCVMORE)
break;
/*
* If we are peeking and the socket receive buffer is
* full, stop since we can't get more data to peek at.
*/
if ((flags & MSG_PEEK) && sbspace(&so->so_rcv) <= 0)
break;
/*
* If we've drained the socket buffer, tell the
* protocol in case it needs to do something to
* get it filled again.
*/
if ((pr->pr_flags & PR_WANTRCVD) && so->so_pcb) {
(*pr->pr_usrreqs->pr_rcvd)(so, flags, l);
}
SBLASTRECORDCHK(&so->so_rcv,
"kttcp_soreceive sbwait 2");
SBLASTMBUFCHK(&so->so_rcv,
"kttcp_soreceive sbwait 2");
error = sbwait(&so->so_rcv);
if (error) {
sbunlock(&so->so_rcv);
sounlock(so);
return (0);
}
if ((m = so->so_rcv.sb_mb) != NULL)
nextrecord = m->m_nextpkt;
}
}
if (m && pr->pr_flags & PR_ATOMIC) {
flags |= MSG_TRUNC;
if ((flags & MSG_PEEK) == 0)
(void) sbdroprecord(&so->so_rcv);
}
if ((flags & MSG_PEEK) == 0) {
if (m == NULL) {
/*
* First part is an SB_EMPTY_FIXUP(). Second part
* makes sure sb_lastrecord is up-to-date if
* there is still data in the socket buffer.
*/
so->so_rcv.sb_mb = nextrecord;
if (so->so_rcv.sb_mb == NULL) {
so->so_rcv.sb_mbtail = NULL;
so->so_rcv.sb_lastrecord = NULL;
} else if (nextrecord->m_nextpkt == NULL)
so->so_rcv.sb_lastrecord = nextrecord;
}
SBLASTRECORDCHK(&so->so_rcv, "kttcp_soreceive 4");
SBLASTMBUFCHK(&so->so_rcv, "kttcp_soreceive 4");
if (pr->pr_flags & PR_WANTRCVD && so->so_pcb) {
(*pr->pr_usrreqs->pr_rcvd)(so, flags, l);
}
}
if (orig_resid == resid && orig_resid &&
(flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) {
sbunlock(&so->so_rcv);
goto restart;
}
if (flagsp)
*flagsp |= flags;
release:
sbunlock(&so->so_rcv);
sounlock(so);
*done = slen - resid;
#if 0
printf("soreceive: error %d slen %llu resid %lld\n", error, slen, resid);
#endif
return (error);
}
/*
* Slightly changed version of sosend()
*/
static int
kttcp_sosend(struct socket *so, unsigned long long slen,
unsigned long long *done, struct lwp *l, int flags)
{
struct mbuf **mp, *m, *top;
long space, len, mlen;
int error, dontroute, atomic;
long long resid;
atomic = sosendallatonce(so);
resid = slen;
top = NULL;
/*
* In theory resid should be unsigned.
* However, space must be signed, as it might be less than 0
* if we over-committed, and we must use a signed comparison
* of space and resid. On the other hand, a negative resid
* causes us to loop sending 0-length segments to the protocol.
*/
if (resid < 0) {
error = EINVAL;
goto out;
}
dontroute =
(flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
(so->so_proto->pr_flags & PR_ATOMIC);
l->l_ru.ru_msgsnd++;
#define snderr(errno) { error = errno; goto release; }
solock(so);
restart:
if ((error = sblock(&so->so_snd, SBLOCKWAIT(flags))) != 0)
goto out;
do {
if (so->so_state & SS_CANTSENDMORE)
snderr(EPIPE);
if (so->so_error) {
error = so->so_error;
so->so_error = 0;
goto release;
}
if ((so->so_state & SS_ISCONNECTED) == 0) {
if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
snderr(ENOTCONN);
} else {
snderr(EDESTADDRREQ);
}
}
space = sbspace(&so->so_snd);
if (flags & MSG_OOB)
space += 1024;
if ((atomic && resid > so->so_snd.sb_hiwat))
snderr(EMSGSIZE);
if (space < resid && (atomic || space < so->so_snd.sb_lowat)) {
if (so->so_state & SS_NBIO)
snderr(EWOULDBLOCK);
SBLASTRECORDCHK(&so->so_rcv,
"kttcp_soreceive sbwait 1");
SBLASTMBUFCHK(&so->so_rcv,
"kttcp_soreceive sbwait 1");
sbunlock(&so->so_snd);
error = sbwait(&so->so_snd);
if (error)
goto out;
goto restart;
}
mp = ⊤
do {
sounlock(so);
do {
if (top == 0) {
m = m_gethdr(M_WAIT, MT_DATA);
mlen = MHLEN;
m->m_pkthdr.len = 0;
m->m_pkthdr.rcvif = NULL;
} else {
m = m_get(M_WAIT, MT_DATA);
mlen = MLEN;
}
if (resid >= MINCLSIZE && space >= MCLBYTES) {
m_clget(m, M_WAIT);
if ((m->m_flags & M_EXT) == 0)
goto nopages;
mlen = MCLBYTES;
#ifdef MAPPED_MBUFS
len = lmin(MCLBYTES, resid);
#else
if (atomic && top == 0) {
len = lmin(MCLBYTES - max_hdr,
resid);
m->m_data += max_hdr;
} else
len = lmin(MCLBYTES, resid);
#endif
space -= len;
} else {
nopages:
len = lmin(lmin(mlen, resid), space);
space -= len;
/*
* For datagram protocols, leave room
* for protocol headers in first mbuf.
*/
if (atomic && top == 0 && len < mlen)
MH_ALIGN(m, len);
}
resid -= len;
m->m_len = len;
*mp = m;
top->m_pkthdr.len += len;
if (error)
goto release;
mp = &m->m_next;
if (resid <= 0) {
if (flags & MSG_EOR)
top->m_flags |= M_EOR;
break;
}
} while (space > 0 && atomic);
solock(so);
if (so->so_state & SS_CANTSENDMORE)
snderr(EPIPE);
if (dontroute)
so->so_options |= SO_DONTROUTE;
if (resid > 0)
so->so_state |= SS_MORETOCOME;
if (flags & MSG_OOB)
error = (*so->so_proto->pr_usrreqs->pr_sendoob)(so,
top, NULL);
else
error = (*so->so_proto->pr_usrreqs->pr_send)(so,
top, NULL, NULL, l);
if (dontroute)
so->so_options &= ~SO_DONTROUTE;
if (resid > 0)
so->so_state &= ~SS_MORETOCOME;
top = 0;
mp = ⊤
if (error)
goto release;
} while (resid && space > 0);
} while (resid);
release:
sbunlock(&so->so_snd);
out:
sounlock(so);
if (top)
m_freem(top);
*done = slen - resid;
#if 0
printf("sosend: error %d slen %llu resid %lld\n", error, slen, resid);
#endif
return (error);
}
static int
handle_ddp(struct socket *so, struct uio *uio, int flags, int error)
{
struct sockbuf *sb = &so->so_rcv;
struct tcpcb *tp = so_sototcpcb(so);
struct toepcb *toep = tp->t_toe;
struct adapter *sc = td_adapter(toep->td);
vm_page_t *pages;
int npages, db_idx, rc, buf_flag;
struct ddp_buffer *db;
struct wrqe *wr;
uint64_t ddp_flags;
SOCKBUF_LOCK_ASSERT(sb);
#if 0
if (sbused(sb) + sc->tt.ddp_thres > uio->uio_resid) {
CTR4(KTR_CXGBE, "%s: sb_cc %d, threshold %d, resid %d",
__func__, sbused(sb), sc->tt.ddp_thres, uio->uio_resid);
}
#endif
/* XXX: too eager to disable DDP, could handle NBIO better than this. */
if (sbused(sb) >= uio->uio_resid || uio->uio_resid < sc->tt.ddp_thres ||
uio->uio_resid > MAX_DDP_BUFFER_SIZE || uio->uio_iovcnt > 1 ||
so->so_state & SS_NBIO || flags & (MSG_DONTWAIT | MSG_NBIO) ||
error || so->so_error || sb->sb_state & SBS_CANTRCVMORE)
goto no_ddp;
/*
* Fault in and then hold the pages of the uio buffers. We'll wire them
* a bit later if everything else works out.
*/
SOCKBUF_UNLOCK(sb);
if (hold_uio(uio, &pages, &npages) != 0) {
SOCKBUF_LOCK(sb);
goto no_ddp;
}
SOCKBUF_LOCK(sb);
if (__predict_false(so->so_error || sb->sb_state & SBS_CANTRCVMORE)) {
vm_page_unhold_pages(pages, npages);
free(pages, M_CXGBE);
goto no_ddp;
}
/*
* Figure out which one of the two DDP buffers to use this time.
*/
db_idx = select_ddp_buffer(sc, toep, pages, npages,
(uintptr_t)uio->uio_iov->iov_base & PAGE_MASK, uio->uio_resid);
pages = NULL; /* handed off to select_ddp_buffer */
if (db_idx < 0)
goto no_ddp;
db = toep->db[db_idx];
buf_flag = db_idx == 0 ? DDP_BUF0_ACTIVE : DDP_BUF1_ACTIVE;
/*
* Build the compound work request that tells the chip where to DMA the
* payload.
*/
ddp_flags = select_ddp_flags(so, flags, db_idx);
wr = mk_update_tcb_for_ddp(sc, toep, db_idx, sbused(sb), ddp_flags);
if (wr == NULL) {
/*
* Just unhold the pages. The DDP buffer's software state is
* left as-is in the toep. The page pods were written
* successfully and we may have an opportunity to use it in the
* future.
*/
vm_page_unhold_pages(db->pages, db->npages);
goto no_ddp;
}
/* Wire (and then unhold) the pages, and give the chip the go-ahead. */
wire_ddp_buffer(db);
t4_wrq_tx(sc, wr);
sb->sb_flags &= ~SB_DDP_INDICATE;
toep->ddp_flags |= buf_flag;
/*
* Wait for the DDP operation to complete and then unwire the pages.
* The return code from the sbwait will be the final return code of this
* function. But we do need to wait for DDP no matter what.
*/
rc = sbwait(sb);
while (toep->ddp_flags & buf_flag) {
/* XXXGL: shouldn't here be sbwait() call? */
sb->sb_flags |= SB_WAIT;
msleep(&sb->sb_acc, &sb->sb_mtx, PSOCK , "sbwait", 0);
}
unwire_ddp_buffer(db);
return (rc);
no_ddp:
disable_ddp(sc, toep);
discourage_ddp(toep);
sb->sb_flags &= ~SB_DDP_INDICATE;
return (0);
}
static int
dhcp_get_offer(struct dhcp_context * context, int wait_ticks)
{
int error = 0;
int gather_count = 0;
const struct in_addr * ip;
int last_rating = 0;
int len;
int n;
int rating;
struct dhcp * reply;
struct in_addr server_id;
struct socket * timer_arg;
timer_arg = context->so;
reply = dhcp_context_reply(context);
timeout((timeout_fcn_t)dhcp_timeout, &timer_arg, wait_ticks);
while (1) {
error = receive_packet(context->so, context->reply,
sizeof(context->reply), &n);
if (error == 0) {
dhcpol_t options;
dprintf(("\ndhcp: received packet length %d\n", n));
if (n < (int)sizeof(struct dhcp)) {
dprintf(("dhcp: packet is too short %d < %d\n",
n, (int)sizeof(struct dhcp)));
continue;
}
if (ntohl(reply->dp_xid) != context->xid
|| reply->dp_yiaddr.s_addr == 0
|| reply->dp_yiaddr.s_addr == INADDR_BROADCAST
|| bcmp(reply->dp_chaddr,
link_address(context->dl_p),
link_address_length(context->dl_p)) != 0) {
/* not for us */
continue;
}
(void)dhcpol_parse_packet(&options, reply, n);
if (get_dhcp_msgtype(&options) != dhcp_msgtype_offer_e) {
/* not an offer */
goto next_packet;
}
ip = (const struct in_addr *)
dhcpol_find(&options,
dhcptag_server_identifier_e, &len, NULL);
if (ip == NULL || len < (int)sizeof(*ip)) {
/* missing/invalid server identifier */
goto next_packet;
}
printf("dhcp: received OFFER: server " IP_FORMAT
" IP address " IP_FORMAT "\n",
IP_LIST(ip), IP_LIST(&reply->dp_yiaddr));
server_id = *ip;
rating = rate_packet(&options);
if (rating > last_rating) {
context->iaddr = reply->dp_yiaddr;
ip = (const struct in_addr *)
dhcpol_find(&options,
dhcptag_subnet_mask_e, &len, NULL);
if (ip != NULL && len >= (int)sizeof(*ip)) {
context->netmask = *ip;
}
ip = (const struct in_addr *)
dhcpol_find(&options, dhcptag_router_e, &len, NULL);
if (ip != NULL && len >= (int)sizeof(*ip)) {
context->router = *ip;
}
context->server_id = server_id;
}
if (rating >= GOOD_RATING) {
dhcpol_free(&options);
/* packet is good enough */
untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
break;
}
if (gather_count == 0) {
untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
timer_arg = context->so;
timeout((timeout_fcn_t)dhcp_timeout, &timer_arg,
hz * GATHER_TIME_SECS);
}
gather_count = 1;
next_packet:
dhcpol_free(&options);
}
else if ((error != EWOULDBLOCK)) {
untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
break;
}
else if (timer_arg == NULL) { /* timed out */
if (gather_count != 0) {
dprintf(("dhcp: gathering time has expired\n"));
error = 0;
}
break;
}
else {
socket_lock(context->so, 1);
error = sbwait(&context->so->so_rcv);
socket_unlock(context->so, 1);
}
}
return (error);
}
static int
dhcp_get_ack(struct dhcp_context * context, int wait_ticks)
{
int error = 0;
const struct in_addr * ip;
int len;
int n;
struct dhcp * reply;
struct in_addr server_id;
struct socket * timer_arg;
timer_arg = context->so;
reply = dhcp_context_reply(context);
timeout((timeout_fcn_t)dhcp_timeout, &timer_arg, wait_ticks);
while (1) {
error = receive_packet(context->so, context->reply,
sizeof(context->reply), &n);
if (error == 0) {
dhcp_msgtype_t msg;
dhcpol_t options;
dprintf(("\ndhcp: received packet length %d\n", n));
if (n < (int)sizeof(struct dhcp)) {
dprintf(("dhcp: packet is too short %d < %d\n",
n, (int)sizeof(struct dhcp)));
continue;
}
if (ntohl(reply->dp_xid) != context->xid
|| bcmp(reply->dp_chaddr, link_address(context->dl_p),
link_address_length(context->dl_p)) != 0) {
/* not for us */
continue;
}
(void)dhcpol_parse_packet(&options, reply, n);
server_id.s_addr = 0;
ip = (const struct in_addr *)
dhcpol_find(&options,
dhcptag_server_identifier_e, &len, NULL);
if (ip != NULL && len >= (int)sizeof(*ip)) {
server_id = *ip;
}
msg = get_dhcp_msgtype(&options);
if (msg == dhcp_msgtype_nak_e
&& server_id.s_addr == context->server_id.s_addr) {
/* server NAK'd us, start over */
dhcpol_free(&options);
error = EPROTO;
untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
break;
}
if (msg != dhcp_msgtype_ack_e
|| reply->dp_yiaddr.s_addr == 0
|| reply->dp_yiaddr.s_addr == INADDR_BROADCAST) {
/* ignore the packet */
goto next_packet;
}
printf("dhcp: received ACK: server " IP_FORMAT
" IP address " IP_FORMAT "\n",
IP_LIST(&server_id), IP_LIST(&reply->dp_yiaddr));
context->iaddr = reply->dp_yiaddr;
ip = (const struct in_addr *)
dhcpol_find(&options,
dhcptag_subnet_mask_e, &len, NULL);
if (ip != NULL && len >= (int)sizeof(*ip)) {
context->netmask = *ip;
}
ip = (const struct in_addr *)
dhcpol_find(&options, dhcptag_router_e, &len, NULL);
if (ip != NULL && len >= (int)sizeof(*ip)) {
context->router = *ip;
}
dhcpol_free(&options);
untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
break;
next_packet:
dhcpol_free(&options);
}
else if ((error != EWOULDBLOCK)) {
/* if some other error occurred, we're done */
untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
break;
}
else if (timer_arg == NULL) {
/* timed out */
break;
}
else {
/* wait for a wait to arrive, or a timeout to occur */
socket_lock(context->so, 1);
error = sbwait(&context->so->so_rcv);
socket_unlock(context->so, 1);
}
}
return (error);
}
