/*
* Tcp output routine: figure out what should be sent and send it.
*/
int
tcp_output(struct tcpcb *tp)
{
struct inpcb * const inp = tp->t_inpcb;
struct socket *so = inp->inp_socket;
long len, recvwin, sendwin;
int nsacked = 0;
int off, flags, error;
#ifdef TCP_SIGNATURE
int sigoff = 0;
#endif
struct mbuf *m;
struct ip *ip = NULL;
struct ipovly *ipov = NULL;
struct tcphdr *th;
u_char opt[TCP_MAXOLEN];
unsigned int ipoptlen, optlen, hdrlen;
int idle;
boolean_t sendalot;
struct ip6_hdr *ip6 = NULL;
#ifdef INET6
const boolean_t isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
#else
const boolean_t isipv6 = FALSE;
#endif
KKASSERT(so->so_port == &curthread->td_msgport);
/*
* Determine length of data that should be transmitted,
* and flags that will be used.
* If there is some data or critical controls (SYN, RST)
* to send, then transmit; otherwise, investigate further.
*/
/*
* If we have been idle for a while, the send congestion window
* could be no longer representative of the current state of the link.
* So unless we are expecting more acks to come in, slow-start from
* scratch to re-determine the send congestion window.
*/
if (tp->snd_max == tp->snd_una &&
(ticks - tp->t_rcvtime) >= tp->t_rxtcur) {
if (tcp_do_rfc3390) {
int initial_cwnd =
min(4 * tp->t_maxseg, max(2 * tp->t_maxseg, 4380));
tp->snd_cwnd = min(tp->snd_cwnd, initial_cwnd);
} else {
tp->snd_cwnd = tp->t_maxseg;
}
tp->snd_wacked = 0;
}
/*
* Calculate whether the transmit stream was previously idle
* and adjust TF_LASTIDLE for the next time.
*/
idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
if (idle && (tp->t_flags & TF_MORETOCOME))
tp->t_flags |= TF_LASTIDLE;
else
tp->t_flags &= ~TF_LASTIDLE;
if (TCP_DO_SACK(tp) && tp->snd_nxt != tp->snd_max &&
!IN_FASTRECOVERY(tp))
nsacked = tcp_sack_bytes_below(&tp->scb, tp->snd_nxt);
again:
/* Make use of SACK information when slow-starting after a RTO. */
if (TCP_DO_SACK(tp) && tp->snd_nxt != tp->snd_max &&
!IN_FASTRECOVERY(tp)) {
tcp_seq old_snd_nxt = tp->snd_nxt;
tcp_sack_skip_sacked(&tp->scb, &tp->snd_nxt);
nsacked += tp->snd_nxt - old_snd_nxt;
}
sendalot = FALSE;
off = tp->snd_nxt - tp->snd_una;
sendwin = min(tp->snd_wnd, tp->snd_cwnd + nsacked);
sendwin = min(sendwin, tp->snd_bwnd);
flags = tcp_outflags[tp->t_state];
/*
* Get standard flags, and add SYN or FIN if requested by 'hidden'
* state flags.
*/
if (tp->t_flags & TF_NEEDFIN)
flags |= TH_FIN;
if (tp->t_flags & TF_NEEDSYN)
flags |= TH_SYN;
/*
* If in persist timeout with window of 0, send 1 byte.
* Otherwise, if window is small but nonzero
* and timer expired, we will send what we can
* and go to transmit state.
*/
if (tp->t_flags & TF_FORCE) {
if (sendwin == 0) {
/*
* If we still have some data to send, then
* clear the FIN bit. Usually this would
* happen below when it realizes that we
* aren't sending all the data. However,
* if we have exactly 1 byte of unsent data,
* then it won't clear the FIN bit below,
* and if we are in persist state, we wind
* up sending the packet without recording
* that we sent the FIN bit.
*
* We can't just blindly clear the FIN bit,
* because if we don't have any more data
* to send then the probe will be the FIN
* itself.
*/
if (off < so->so_snd.ssb_cc)
flags &= ~TH_FIN;
sendwin = 1;
} else {
tcp_callout_stop(tp, tp->tt_persist);
tp->t_rxtshift = 0;
}
}
/*
* If snd_nxt == snd_max and we have transmitted a FIN, the
* offset will be > 0 even if so_snd.ssb_cc is 0, resulting in
* a negative length. This can also occur when TCP opens up
* its congestion window while receiving additional duplicate
* acks after fast-retransmit because TCP will reset snd_nxt
* to snd_max after the fast-retransmit.
*
* In the normal retransmit-FIN-only case, however, snd_nxt will
* be set to snd_una, the offset will be 0, and the length may
* wind up 0.
*/
len = (long)ulmin(so->so_snd.ssb_cc, sendwin) - off;
/*
* Lop off SYN bit if it has already been sent. However, if this
* is SYN-SENT state and if segment contains data, suppress sending
* segment (sending the segment would be an option if we still
* did TAO and the remote host supported it).
*/
if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
flags &= ~TH_SYN;
off--, len++;
if (len > 0 && tp->t_state == TCPS_SYN_SENT)
return 0;
}
/*
* Be careful not to send data and/or FIN on SYN segments.
* This measure is needed to prevent interoperability problems
* with not fully conformant TCP implementations.
*/
if (flags & TH_SYN) {
len = 0;
flags &= ~TH_FIN;
}
if (len < 0) {
/*
* If FIN has been sent but not acked,
* but we haven't been called to retransmit,
* len will be < 0. Otherwise, window shrank
* after we sent into it. If window shrank to 0,
* cancel pending retransmit, pull snd_nxt back
* to (closed) window, and set the persist timer
* if it isn't already going. If the window didn't
* close completely, just wait for an ACK.
*/
len = 0;
if (sendwin == 0) {
tcp_callout_stop(tp, tp->tt_rexmt);
tp->t_rxtshift = 0;
tp->snd_nxt = tp->snd_una;
if (!tcp_callout_active(tp, tp->tt_persist))
tcp_setpersist(tp);
}
}
KASSERT(len >= 0, ("%s: len < 0", __func__));
/*
* Automatic sizing of send socket buffer. Often the send buffer
* size is not optimally adjusted to the actual network conditions
* at hand (delay bandwidth product). Setting the buffer size too
* small limits throughput on links with high bandwidth and high
* delay (eg. trans-continental/oceanic links). Setting the
* buffer size too big consumes too much real kernel memory,
* especially with many connections on busy servers.
*
* The criteria to step up the send buffer one notch are:
* 1. receive window of remote host is larger than send buffer
* (with a fudge factor of 5/4th);
* 2. send buffer is filled to 7/8th with data (so we actually
* have data to make use of it);
* 3. send buffer fill has not hit maximal automatic size;
* 4. our send window (slow start and cogestion controlled) is
* larger than sent but unacknowledged data in send buffer.
*
* The remote host receive window scaling factor may limit the
* growing of the send buffer before it reaches its allowed
* maximum.
*
* It scales directly with slow start or congestion window
* and does at most one step per received ACK. This fast
* scaling has the drawback of growing the send buffer beyond
* what is strictly necessary to make full use of a given
* delay*bandwith product. However testing has shown this not
* to be much of an problem. At worst we are trading wasting
* of available bandwith (the non-use of it) for wasting some
* socket buffer memory.
*
* TODO: Shrink send buffer during idle periods together
* with congestion window. Requires another timer. Has to
* wait for upcoming tcp timer rewrite.
*/
if (tcp_do_autosndbuf && so->so_snd.ssb_flags & SSB_AUTOSIZE) {
if ((tp->snd_wnd / 4 * 5) >= so->so_snd.ssb_hiwat &&
so->so_snd.ssb_cc >= (so->so_snd.ssb_hiwat / 8 * 7) &&
so->so_snd.ssb_cc < tcp_autosndbuf_max &&
sendwin >= (so->so_snd.ssb_cc - (tp->snd_nxt - tp->snd_una))) {
u_long newsize;
newsize = ulmin(so->so_snd.ssb_hiwat +
tcp_autosndbuf_inc,
tcp_autosndbuf_max);
if (!ssb_reserve(&so->so_snd, newsize, so, NULL))
atomic_clear_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
if (newsize >= (TCP_MAXWIN << tp->snd_scale))
atomic_clear_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
}
}
/*
* Truncate to the maximum segment length and ensure that FIN is
* removed if the length no longer contains the last data byte.
*/
if (len > tp->t_maxseg) {
len = tp->t_maxseg;
sendalot = TRUE;
}
if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.ssb_cc))
flags &= ~TH_FIN;
recvwin = ssb_space(&so->so_rcv);
/*
* Sender silly window avoidance. We transmit under the following
* conditions when len is non-zero:
*
* - We have a full segment
* - This is the last buffer in a write()/send() and we are
* either idle or running NODELAY
* - we've timed out (e.g. persist timer)
* - we have more then 1/2 the maximum send window's worth of
* data (receiver may be limiting the window size)
* - we need to retransmit
*/
if (len) {
if (len == tp->t_maxseg)
goto send;
/*
* NOTE! on localhost connections an 'ack' from the remote
* end may occur synchronously with the output and cause
* us to flush a buffer queued with moretocome. XXX
*
* note: the len + off check is almost certainly unnecessary.
*/
if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */
(idle || (tp->t_flags & TF_NODELAY)) &&
len + off >= so->so_snd.ssb_cc &&
!(tp->t_flags & TF_NOPUSH)) {
goto send;
}
if (tp->t_flags & TF_FORCE) /* typ. timeout case */
goto send;
if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
goto send;
if (SEQ_LT(tp->snd_nxt, tp->snd_max)) /* retransmit case */
goto send;
}
/*
* Compare available window to amount of window
* known to peer (as advertised window less
* next expected input). If the difference is at least two
* max size segments, or at least 50% of the maximum possible
* window, then want to send a window update to peer.
*/
if (recvwin > 0) {
/*
* "adv" is the amount we can increase the window,
* taking into account that we are limited by
* TCP_MAXWIN << tp->rcv_scale.
*/
long adv = min(recvwin, (long)TCP_MAXWIN << tp->rcv_scale) -
(tp->rcv_adv - tp->rcv_nxt);
long hiwat;
/*
* This ack case typically occurs when the user has drained
* the TCP socket buffer sufficiently to warrent an ack
* containing a 'pure window update'... that is, an ack that
* ONLY updates the tcp window.
*
* It is unclear why we would need to do a pure window update
* past 2 segments if we are going to do one at 1/2 the high
* water mark anyway, especially since under normal conditions
* the user program will drain the socket buffer quickly.
* The 2-segment pure window update will often add a large
* number of extra, unnecessary acks to the stream.
*
* avoid_pure_win_update now defaults to 1.
*/
if (avoid_pure_win_update == 0 ||
(tp->t_flags & TF_RXRESIZED)) {
if (adv >= (long) (2 * tp->t_maxseg)) {
goto send;
}
}
hiwat = (long)(TCP_MAXWIN << tp->rcv_scale);
if (hiwat > (long)so->so_rcv.ssb_hiwat)
hiwat = (long)so->so_rcv.ssb_hiwat;
if (adv >= hiwat / 2)
goto send;
}
/*
* Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
* is also a catch-all for the retransmit timer timeout case.
*/
if (tp->t_flags & TF_ACKNOW)
goto send;
if ((flags & TH_RST) ||
((flags & TH_SYN) && !(tp->t_flags & TF_NEEDSYN)))
goto send;
if (SEQ_GT(tp->snd_up, tp->snd_una))
goto send;
/*
* If our state indicates that FIN should be sent
* and we have not yet done so, then we need to send.
*/
if (flags & TH_FIN &&
(!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una))
goto send;
/*
* TCP window updates are not reliable, rather a polling protocol
* using ``persist'' packets is used to insure receipt of window
* updates. The three ``states'' for the output side are:
* idle not doing retransmits or persists
* persisting to move a small or zero window
* (re)transmitting and thereby not persisting
*
* tcp_callout_active(tp, tp->tt_persist)
* is true when we are in persist state.
* The TF_FORCE flag in tp->t_flags
* is set when we are called to send a persist packet.
* tcp_callout_active(tp, tp->tt_rexmt)
* is set when we are retransmitting
* The output side is idle when both timers are zero.
*
* If send window is too small, there is data to transmit, and no
* retransmit or persist is pending, then go to persist state.
* If nothing happens soon, send when timer expires:
* if window is nonzero, transmit what we can,
* otherwise force out a byte.
*/
if (so->so_snd.ssb_cc > 0 &&
!tcp_callout_active(tp, tp->tt_rexmt) &&
!tcp_callout_active(tp, tp->tt_persist)) {
tp->t_rxtshift = 0;
tcp_setpersist(tp);
}
/*
* No reason to send a segment, just return.
*/
return (0);
send:
/*
* Before ESTABLISHED, force sending of initial options
* unless TCP set not to do any options.
* NOTE: we assume that the IP/TCP header plus TCP options
* always fit in a single mbuf, leaving room for a maximum
* link header, i.e.
* max_linkhdr + sizeof(struct tcpiphdr) + optlen <= MCLBYTES
*/
optlen = 0;
if (isipv6)
hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
else
hdrlen = sizeof(struct tcpiphdr);
if (flags & TH_SYN) {
tp->snd_nxt = tp->iss;
if (!(tp->t_flags & TF_NOOPT)) {
u_short mss;
opt[0] = TCPOPT_MAXSEG;
opt[1] = TCPOLEN_MAXSEG;
mss = htons((u_short) tcp_mssopt(tp));
memcpy(opt + 2, &mss, sizeof mss);
optlen = TCPOLEN_MAXSEG;
if ((tp->t_flags & TF_REQ_SCALE) &&
(!(flags & TH_ACK) ||
(tp->t_flags & TF_RCVD_SCALE))) {
*((u_int32_t *)(opt + optlen)) = htonl(
TCPOPT_NOP << 24 |
TCPOPT_WINDOW << 16 |
TCPOLEN_WINDOW << 8 |
tp->request_r_scale);
optlen += 4;
}
if ((tcp_do_sack && !(flags & TH_ACK)) ||
tp->t_flags & TF_SACK_PERMITTED) {
uint32_t *lp = (uint32_t *)(opt + optlen);
*lp = htonl(TCPOPT_SACK_PERMITTED_ALIGNED);
optlen += TCPOLEN_SACK_PERMITTED_ALIGNED;
}
}
}
/*
* Send a timestamp and echo-reply if this is a SYN and our side
* wants to use timestamps (TF_REQ_TSTMP is set) or both our side
* and our peer have sent timestamps in our SYN's.
*/
if ((tp->t_flags & (TF_REQ_TSTMP | TF_NOOPT)) == TF_REQ_TSTMP &&
!(flags & TH_RST) &&
(!(flags & TH_ACK) || (tp->t_flags & TF_RCVD_TSTMP))) {
u_int32_t *lp = (u_int32_t *)(opt + optlen);
/* Form timestamp option as shown in appendix A of RFC 1323. */
*lp++ = htonl(TCPOPT_TSTAMP_HDR);
*lp++ = htonl(ticks);
*lp = htonl(tp->ts_recent);
optlen += TCPOLEN_TSTAMP_APPA;
}
/* Set receive buffer autosizing timestamp. */
if (tp->rfbuf_ts == 0 && (so->so_rcv.ssb_flags & SSB_AUTOSIZE))
tp->rfbuf_ts = ticks;
/*
* If this is a SACK connection and we have a block to report,
* fill in the SACK blocks in the TCP options.
*/
if ((tp->t_flags & (TF_SACK_PERMITTED | TF_NOOPT)) ==
TF_SACK_PERMITTED &&
(!LIST_EMPTY(&tp->t_segq) ||
tp->reportblk.rblk_start != tp->reportblk.rblk_end))
tcp_sack_fill_report(tp, opt, &optlen);
#ifdef TCP_SIGNATURE
if (tp->t_flags & TF_SIGNATURE) {
int i;
u_char *bp;
/*
* Initialize TCP-MD5 option (RFC2385)
*/
bp = (u_char *)opt + optlen;
*bp++ = TCPOPT_SIGNATURE;
*bp++ = TCPOLEN_SIGNATURE;
sigoff = optlen + 2;
for (i = 0; i < TCP_SIGLEN; i++)
*bp++ = 0;
optlen += TCPOLEN_SIGNATURE;
/*
* Terminate options list and maintain 32-bit alignment.
*/
*bp++ = TCPOPT_NOP;
*bp++ = TCPOPT_EOL;
optlen += 2;
}
#endif /* TCP_SIGNATURE */
KASSERT(optlen <= TCP_MAXOLEN, ("too many TCP options"));
hdrlen += optlen;
if (isipv6) {
ipoptlen = ip6_optlen(inp);
} else {
if (inp->inp_options) {
ipoptlen = inp->inp_options->m_len -
offsetof(struct ipoption, ipopt_list);
} else {
/*
* Tcp output routine: figure out what should be sent and send it.
*/
int
tcp_output(struct tcpcb *tp)
{
struct inpcb * const inp = tp->t_inpcb;
struct socket *so = inp->inp_socket;
long len, recvwin, sendwin;
int nsacked = 0;
int off, flags, error = 0;
#ifdef TCP_SIGNATURE
int sigoff = 0;
#endif
struct mbuf *m;
struct ip *ip;
struct tcphdr *th;
u_char opt[TCP_MAXOLEN];
unsigned int ipoptlen, optlen, hdrlen;
int idle;
boolean_t sendalot;
struct ip6_hdr *ip6;
#ifdef INET6
const boolean_t isipv6 = INP_ISIPV6(inp);
#else
const boolean_t isipv6 = FALSE;
#endif
boolean_t can_tso = FALSE, use_tso;
boolean_t report_sack, idle_cwv = FALSE;
u_int segsz, tso_hlen, tso_lenmax = 0;
int segcnt = 0;
boolean_t need_sched = FALSE;
KKASSERT(so->so_port == &curthread->td_msgport);
/*
* Determine length of data that should be transmitted,
* and flags that will be used.
* If there is some data or critical controls (SYN, RST)
* to send, then transmit; otherwise, investigate further.
*/
/*
* If we have been idle for a while, the send congestion window
* could be no longer representative of the current state of the
* link; need to validate congestion window. However, we should
* not perform congestion window validation here, since we could
* be asked to send pure ACK.
*/
if (tp->snd_max == tp->snd_una &&
(ticks - tp->snd_last) >= tp->t_rxtcur && tcp_idle_restart)
idle_cwv = TRUE;
/*
* Calculate whether the transmit stream was previously idle
* and adjust TF_LASTIDLE for the next time.
*/
idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
if (idle && (tp->t_flags & TF_MORETOCOME))
tp->t_flags |= TF_LASTIDLE;
else
tp->t_flags &= ~TF_LASTIDLE;
if (TCP_DO_SACK(tp) && tp->snd_nxt != tp->snd_max &&
!IN_FASTRECOVERY(tp))
nsacked = tcp_sack_bytes_below(&tp->scb, tp->snd_nxt);
/*
* Find out whether TSO could be used or not
*
* For TSO capable devices, the following assumptions apply to
* the processing of TCP flags:
* - If FIN is set on the large TCP segment, the device must set
* FIN on the last segment that it creates from the large TCP
* segment.
* - If PUSH is set on the large TCP segment, the device must set
* PUSH on the last segment that it creates from the large TCP
* segment.
*/
#if !defined(IPSEC) && !defined(FAST_IPSEC)
if (tcp_do_tso
#ifdef TCP_SIGNATURE
&& (tp->t_flags & TF_SIGNATURE) == 0
#endif
) {
if (!isipv6) {
struct rtentry *rt = inp->inp_route.ro_rt;
if (rt != NULL && (rt->rt_flags & RTF_UP) &&
(rt->rt_ifp->if_hwassist & CSUM_TSO)) {
can_tso = TRUE;
tso_lenmax = rt->rt_ifp->if_tsolen;
}
}
}
#endif /* !IPSEC && !FAST_IPSEC */
again:
m = NULL;
ip = NULL;
th = NULL;
ip6 = NULL;
if ((tp->t_flags & (TF_SACK_PERMITTED | TF_NOOPT)) ==
TF_SACK_PERMITTED &&
(!TAILQ_EMPTY(&tp->t_segq) ||
tp->reportblk.rblk_start != tp->reportblk.rblk_end))
report_sack = TRUE;
else
report_sack = FALSE;
/* Make use of SACK information when slow-starting after a RTO. */
if (TCP_DO_SACK(tp) && tp->snd_nxt != tp->snd_max &&
!IN_FASTRECOVERY(tp)) {
tcp_seq old_snd_nxt = tp->snd_nxt;
tcp_sack_skip_sacked(&tp->scb, &tp->snd_nxt);
nsacked += tp->snd_nxt - old_snd_nxt;
}
sendalot = FALSE;
off = tp->snd_nxt - tp->snd_una;
sendwin = min(tp->snd_wnd, tp->snd_cwnd + nsacked);
sendwin = min(sendwin, tp->snd_bwnd);
flags = tcp_outflags[tp->t_state];
/*
* Get standard flags, and add SYN or FIN if requested by 'hidden'
* state flags.
*/
if (tp->t_flags & TF_NEEDFIN)
flags |= TH_FIN;
if (tp->t_flags & TF_NEEDSYN)
flags |= TH_SYN;
/*
* If in persist timeout with window of 0, send 1 byte.
* Otherwise, if window is small but nonzero
* and timer expired, we will send what we can
* and go to transmit state.
*/
if (tp->t_flags & TF_FORCE) {
if (sendwin == 0) {
/*
* If we still have some data to send, then
* clear the FIN bit. Usually this would
* happen below when it realizes that we
* aren't sending all the data. However,
* if we have exactly 1 byte of unsent data,
* then it won't clear the FIN bit below,
* and if we are in persist state, we wind
* up sending the packet without recording
* that we sent the FIN bit.
*
* We can't just blindly clear the FIN bit,
* because if we don't have any more data
* to send then the probe will be the FIN
* itself.
*/
if (off < so->so_snd.ssb_cc)
flags &= ~TH_FIN;
sendwin = 1;
} else {
tcp_callout_stop(tp, tp->tt_persist);
tp->t_rxtshift = 0;
}
}
/*
* If snd_nxt == snd_max and we have transmitted a FIN, the
* offset will be > 0 even if so_snd.ssb_cc is 0, resulting in
* a negative length. This can also occur when TCP opens up
* its congestion window while receiving additional duplicate
* acks after fast-retransmit because TCP will reset snd_nxt
* to snd_max after the fast-retransmit.
*
* A negative length can also occur when we are in the
* TCPS_SYN_RECEIVED state due to a simultanious connect where
* our SYN has not been acked yet.
*
* In the normal retransmit-FIN-only case, however, snd_nxt will
* be set to snd_una, the offset will be 0, and the length may
* wind up 0.
*/
len = (long)ulmin(so->so_snd.ssb_cc, sendwin) - off;
/*
* Lop off SYN bit if it has already been sent. However, if this
* is SYN-SENT state and if segment contains data, suppress sending
* segment (sending the segment would be an option if we still
* did TAO and the remote host supported it).
*/
if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
flags &= ~TH_SYN;
off--, len++;
if (len > 0 && tp->t_state == TCPS_SYN_SENT) {
tp->t_flags &= ~(TF_ACKNOW | TF_XMITNOW);
return 0;
}
}
/*
* Be careful not to send data and/or FIN on SYN segments.
* This measure is needed to prevent interoperability problems
* with not fully conformant TCP implementations.
*/
if (flags & TH_SYN) {
len = 0;
flags &= ~TH_FIN;
}
if (len < 0) {
/*
* A negative len can occur if our FIN has been sent but not
* acked, or if we are in a simultanious connect in the
* TCPS_SYN_RECEIVED state with our SYN sent but not yet
* acked.
*
* If our window has contracted to 0 in the FIN case
* (which can only occur if we have NOT been called to
* retransmit as per code a few paragraphs up) then we
* want to shift the retransmit timer over to the
* persist timer.
*
* However, if we are in the TCPS_SYN_RECEIVED state
* (the SYN case) we will be in a simultanious connect and
* the window may be zero degeneratively. In this case we
* do not want to shift to the persist timer after the SYN
* or the SYN+ACK transmission.
*/
len = 0;
if (sendwin == 0 && tp->t_state != TCPS_SYN_RECEIVED) {
tcp_callout_stop(tp, tp->tt_rexmt);
tp->t_rxtshift = 0;
tp->snd_nxt = tp->snd_una;
if (!tcp_callout_active(tp, tp->tt_persist))
tcp_setpersist(tp);
}
}
KASSERT(len >= 0, ("%s: len < 0", __func__));
/*
* Automatic sizing of send socket buffer. Often the send buffer
* size is not optimally adjusted to the actual network conditions
* at hand (delay bandwidth product). Setting the buffer size too
* small limits throughput on links with high bandwidth and high
* delay (eg. trans-continental/oceanic links). Setting the
* buffer size too big consumes too much real kernel memory,
* especially with many connections on busy servers.
*
* The criteria to step up the send buffer one notch are:
* 1. receive window of remote host is larger than send buffer
* (with a fudge factor of 5/4th);
* 2. hiwat has not significantly exceeded bwnd (inflight)
* (bwnd is a maximal value if inflight is disabled).
* 3. send buffer is filled to 7/8th with data (so we actually
* have data to make use of it);
* 4. hiwat has not hit maximal automatic size;
* 5. our send window (slow start and cogestion controlled) is
* larger than sent but unacknowledged data in send buffer.
*
* The remote host receive window scaling factor may limit the
* growing of the send buffer before it reaches its allowed
* maximum.
*
* It scales directly with slow start or congestion window
* and does at most one step per received ACK. This fast
* scaling has the drawback of growing the send buffer beyond
* what is strictly necessary to make full use of a given
* delay*bandwith product. However testing has shown this not
* to be much of an problem. At worst we are trading wasting
* of available bandwith (the non-use of it) for wasting some
* socket buffer memory.
*
* The criteria for shrinking the buffer is based solely on
* the inflight code (snd_bwnd). If inflight is disabled,
* the buffer will not be shrinked. Note that snd_bwnd already
* has a fudge factor. Our test adds a little hysteresis.
*/
if (tcp_do_autosndbuf && (so->so_snd.ssb_flags & SSB_AUTOSIZE)) {
const int asbinc = tcp_autosndbuf_inc;
const int hiwat = so->so_snd.ssb_hiwat;
const int lowat = so->so_snd.ssb_lowat;
u_long newsize;
if ((tp->snd_wnd / 4 * 5) >= hiwat &&
so->so_snd.ssb_cc >= (hiwat / 8 * 7) &&
hiwat < tp->snd_bwnd + hiwat / 10 &&
hiwat + asbinc < tcp_autosndbuf_max &&
hiwat < (TCP_MAXWIN << tp->snd_scale) &&
sendwin >= (so->so_snd.ssb_cc -
(tp->snd_nxt - tp->snd_una))) {
newsize = ulmin(hiwat + asbinc, tcp_autosndbuf_max);
if (!ssb_reserve(&so->so_snd, newsize, so, NULL))
atomic_clear_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
#if 0
if (newsize >= (TCP_MAXWIN << tp->snd_scale))
atomic_clear_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
#endif
} else if ((long)tp->snd_bwnd <
(long)(hiwat * 3 / 4 - lowat - asbinc) &&
hiwat > tp->t_maxseg * 2 + asbinc &&
hiwat + asbinc >= tcp_autosndbuf_min &&
tcp_do_autosndbuf == 1) {
newsize = ulmax(hiwat - asbinc, tp->t_maxseg * 2);
ssb_reserve(&so->so_snd, newsize, so, NULL);
}
}
/*
* Don't use TSO, if:
* - Congestion window needs validation
* - There are SACK blocks to report
* - RST or SYN flags is set
* - URG will be set
*
* XXX
* Checking for SYN|RST looks overkill, just to be safe than sorry
*/
use_tso = can_tso;
if (report_sack || idle_cwv || (flags & (TH_RST | TH_SYN)))
use_tso = FALSE;
if (use_tso) {
tcp_seq ugr_nxt = tp->snd_nxt;
if ((flags & TH_FIN) && (tp->t_flags & TF_SENTFIN) &&
tp->snd_nxt == tp->snd_max)
--ugr_nxt;
if (SEQ_GT(tp->snd_up, ugr_nxt))
use_tso = FALSE;
}
if (use_tso) {
/*
* Find out segment size and header length for TSO
*/
error = tcp_tso_getsize(tp, &segsz, &tso_hlen);
if (error)
use_tso = FALSE;
}
if (!use_tso) {
segsz = tp->t_maxseg;
tso_hlen = 0; /* not used */
}
/*
* Truncate to the maximum segment length if not TSO, and ensure that
* FIN is removed if the length no longer contains the last data byte.
*/
if (len > segsz) {
if (!use_tso) {
len = segsz;
++segcnt;
} else {
int nsegs;
if (__predict_false(tso_lenmax < segsz))
tso_lenmax = segsz << 1;
/*
* Truncate TSO transfers to (IP_MAXPACKET - iphlen -
* thoff), and make sure that we send equal size
* transfers down the stack (rather than big-small-
* big-small-...).
*/
len = min(len, tso_lenmax);
nsegs = min(len, (IP_MAXPACKET - tso_hlen)) / segsz;
KKASSERT(nsegs > 0);
len = nsegs * segsz;
if (len <= segsz) {
use_tso = FALSE;
++segcnt;
} else {
segcnt += nsegs;
}
}
sendalot = TRUE;
} else {
use_tso = FALSE;
if (len > 0)
++segcnt;
}
if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.ssb_cc))
flags &= ~TH_FIN;
recvwin = ssb_space(&so->so_rcv);
/*
* Sender silly window avoidance. We transmit under the following
* conditions when len is non-zero:
*
* - We have a full segment
* - This is the last buffer in a write()/send() and we are
* either idle or running NODELAY
* - we've timed out (e.g. persist timer)
* - we have more then 1/2 the maximum send window's worth of
* data (receiver may be limiting the window size)
* - we need to retransmit
*/
if (len) {
if (len >= segsz)
goto send;
/*
* NOTE! on localhost connections an 'ack' from the remote
* end may occur synchronously with the output and cause
* us to flush a buffer queued with moretocome. XXX
*
* note: the len + off check is almost certainly unnecessary.
*/
if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */
(idle || (tp->t_flags & TF_NODELAY)) &&
len + off >= so->so_snd.ssb_cc &&
!(tp->t_flags & TF_NOPUSH)) {
goto send;
}
if (tp->t_flags & TF_FORCE) /* typ. timeout case */
goto send;
if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
goto send;
if (SEQ_LT(tp->snd_nxt, tp->snd_max)) /* retransmit case */
goto send;
if (tp->t_flags & TF_XMITNOW)
goto send;
}
/*
* Compare available window to amount of window
* known to peer (as advertised window less
* next expected input). If the difference is at least two
* max size segments, or at least 50% of the maximum possible
* window, then want to send a window update to peer.
*/
if (recvwin > 0) {
/*
* "adv" is the amount we can increase the window,
* taking into account that we are limited by
* TCP_MAXWIN << tp->rcv_scale.
*/
long adv = min(recvwin, (long)TCP_MAXWIN << tp->rcv_scale) -
(tp->rcv_adv - tp->rcv_nxt);
long hiwat;
/*
* This ack case typically occurs when the user has drained
* the TCP socket buffer sufficiently to warrent an ack
* containing a 'pure window update'... that is, an ack that
* ONLY updates the tcp window.
*
* It is unclear why we would need to do a pure window update
* past 2 segments if we are going to do one at 1/2 the high
* water mark anyway, especially since under normal conditions
* the user program will drain the socket buffer quickly.
* The 2-segment pure window update will often add a large
* number of extra, unnecessary acks to the stream.
*
* avoid_pure_win_update now defaults to 1.
*/
if (avoid_pure_win_update == 0 ||
(tp->t_flags & TF_RXRESIZED)) {
if (adv >= (long) (2 * segsz)) {
goto send;
}
}
hiwat = (long)(TCP_MAXWIN << tp->rcv_scale);
if (hiwat > (long)so->so_rcv.ssb_hiwat)
hiwat = (long)so->so_rcv.ssb_hiwat;
if (adv >= hiwat / 2)
goto send;
}
/*
* Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
* is also a catch-all for the retransmit timer timeout case.
*/
if (tp->t_flags & TF_ACKNOW)
goto send;
if ((flags & TH_RST) ||
((flags & TH_SYN) && !(tp->t_flags & TF_NEEDSYN)))
goto send;
if (SEQ_GT(tp->snd_up, tp->snd_una))
goto send;
/*
* If our state indicates that FIN should be sent
* and we have not yet done so, then we need to send.
*/
if ((flags & TH_FIN) &&
(!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una))
goto send;
/*
* TCP window updates are not reliable, rather a polling protocol
* using ``persist'' packets is used to insure receipt of window
* updates. The three ``states'' for the output side are:
* idle not doing retransmits or persists
* persisting to move a small or zero window
* (re)transmitting and thereby not persisting
*
* tcp_callout_active(tp, tp->tt_persist)
* is true when we are in persist state.
* The TF_FORCE flag in tp->t_flags
* is set when we are called to send a persist packet.
* tcp_callout_active(tp, tp->tt_rexmt)
* is set when we are retransmitting
* The output side is idle when both timers are zero.
*
* If send window is too small, there is data to transmit, and no
* retransmit or persist is pending, then go to persist state.
*
* If nothing happens soon, send when timer expires:
* if window is nonzero, transmit what we can, otherwise force out
* a byte.
*
* Don't try to set the persist state if we are in TCPS_SYN_RECEIVED
* with data pending. This situation can occur during a
* simultanious connect.
*/
if (so->so_snd.ssb_cc > 0 &&
tp->t_state != TCPS_SYN_RECEIVED &&
!tcp_callout_active(tp, tp->tt_rexmt) &&
!tcp_callout_active(tp, tp->tt_persist)) {
tp->t_rxtshift = 0;
tcp_setpersist(tp);
}
/*
* No reason to send a segment, just return.
*/
tp->t_flags &= ~TF_XMITNOW;
return (0);
send:
if (need_sched && len > 0) {
tcp_output_sched(tp);
return 0;
}
/*
* Before ESTABLISHED, force sending of initial options
* unless TCP set not to do any options.
* NOTE: we assume that the IP/TCP header plus TCP options
* always fit in a single mbuf, leaving room for a maximum
* link header, i.e.
* max_linkhdr + sizeof(struct tcpiphdr) + optlen <= MCLBYTES
*/
optlen = 0;
if (isipv6)
hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
else
hdrlen = sizeof(struct tcpiphdr);
if (flags & TH_SYN) {
tp->snd_nxt = tp->iss;
if (!(tp->t_flags & TF_NOOPT)) {
u_short mss;
opt[0] = TCPOPT_MAXSEG;
opt[1] = TCPOLEN_MAXSEG;
mss = htons((u_short) tcp_mssopt(tp));
memcpy(opt + 2, &mss, sizeof mss);
optlen = TCPOLEN_MAXSEG;
if ((tp->t_flags & TF_REQ_SCALE) &&
(!(flags & TH_ACK) ||
(tp->t_flags & TF_RCVD_SCALE))) {
*((u_int32_t *)(opt + optlen)) = htonl(
TCPOPT_NOP << 24 |
TCPOPT_WINDOW << 16 |
TCPOLEN_WINDOW << 8 |
tp->request_r_scale);
optlen += 4;
}
if ((tcp_do_sack && !(flags & TH_ACK)) ||
tp->t_flags & TF_SACK_PERMITTED) {
uint32_t *lp = (uint32_t *)(opt + optlen);
*lp = htonl(TCPOPT_SACK_PERMITTED_ALIGNED);
optlen += TCPOLEN_SACK_PERMITTED_ALIGNED;
}
}
}
/*
* Send a timestamp and echo-reply if this is a SYN and our side
* wants to use timestamps (TF_REQ_TSTMP is set) or both our side
* and our peer have sent timestamps in our SYN's.
*/
if ((tp->t_flags & (TF_REQ_TSTMP | TF_NOOPT)) == TF_REQ_TSTMP &&
!(flags & TH_RST) &&
(!(flags & TH_ACK) || (tp->t_flags & TF_RCVD_TSTMP))) {
u_int32_t *lp = (u_int32_t *)(opt + optlen);
/* Form timestamp option as shown in appendix A of RFC 1323. */
*lp++ = htonl(TCPOPT_TSTAMP_HDR);
*lp++ = htonl(ticks);
*lp = htonl(tp->ts_recent);
optlen += TCPOLEN_TSTAMP_APPA;
}
/* Set receive buffer autosizing timestamp. */
if (tp->rfbuf_ts == 0 && (so->so_rcv.ssb_flags & SSB_AUTOSIZE))
tp->rfbuf_ts = ticks;
/*
* If this is a SACK connection and we have a block to report,
* fill in the SACK blocks in the TCP options.
*/
if (report_sack)
tcp_sack_fill_report(tp, opt, &optlen);
#ifdef TCP_SIGNATURE
if (tp->t_flags & TF_SIGNATURE) {
int i;
u_char *bp;
/*
* Initialize TCP-MD5 option (RFC2385)
*/
bp = (u_char *)opt + optlen;
*bp++ = TCPOPT_SIGNATURE;
*bp++ = TCPOLEN_SIGNATURE;
sigoff = optlen + 2;
for (i = 0; i < TCP_SIGLEN; i++)
*bp++ = 0;
optlen += TCPOLEN_SIGNATURE;
/*
* Terminate options list and maintain 32-bit alignment.
*/
*bp++ = TCPOPT_NOP;
*bp++ = TCPOPT_EOL;
optlen += 2;
}
#endif /* TCP_SIGNATURE */
KASSERT(optlen <= TCP_MAXOLEN, ("too many TCP options"));
hdrlen += optlen;
if (isipv6) {
ipoptlen = ip6_optlen(inp);
} else {
if (inp->inp_options) {
ipoptlen = inp->inp_options->m_len -
offsetof(struct ipoption, ipopt_list);
} else {