/*
* Tcp output routine: figure out what should be sent and send it.
*/
int
tcp_output(struct tcpcb *tp)
{
struct inpcb * const inp = tp->t_inpcb;
struct socket *so = inp->inp_socket;
long len, recvwin, sendwin;
int nsacked = 0;
int off, flags, error;
#ifdef TCP_SIGNATURE
int sigoff = 0;
#endif
struct mbuf *m;
struct ip *ip = NULL;
struct ipovly *ipov = NULL;
struct tcphdr *th;
u_char opt[TCP_MAXOLEN];
unsigned int ipoptlen, optlen, hdrlen;
int idle;
boolean_t sendalot;
struct ip6_hdr *ip6 = NULL;
#ifdef INET6
const boolean_t isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
#else
const boolean_t isipv6 = FALSE;
#endif
KKASSERT(so->so_port == &curthread->td_msgport);
/*
* Determine length of data that should be transmitted,
* and flags that will be used.
* If there is some data or critical controls (SYN, RST)
* to send, then transmit; otherwise, investigate further.
*/
/*
* If we have been idle for a while, the send congestion window
* could be no longer representative of the current state of the link.
* So unless we are expecting more acks to come in, slow-start from
* scratch to re-determine the send congestion window.
*/
if (tp->snd_max == tp->snd_una &&
(ticks - tp->t_rcvtime) >= tp->t_rxtcur) {
if (tcp_do_rfc3390) {
int initial_cwnd =
min(4 * tp->t_maxseg, max(2 * tp->t_maxseg, 4380));
tp->snd_cwnd = min(tp->snd_cwnd, initial_cwnd);
} else {
tp->snd_cwnd = tp->t_maxseg;
}
tp->snd_wacked = 0;
}
/*
* Calculate whether the transmit stream was previously idle
* and adjust TF_LASTIDLE for the next time.
*/
idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
if (idle && (tp->t_flags & TF_MORETOCOME))
tp->t_flags |= TF_LASTIDLE;
else
tp->t_flags &= ~TF_LASTIDLE;
if (TCP_DO_SACK(tp) && tp->snd_nxt != tp->snd_max &&
!IN_FASTRECOVERY(tp))
nsacked = tcp_sack_bytes_below(&tp->scb, tp->snd_nxt);
again:
/* Make use of SACK information when slow-starting after a RTO. */
if (TCP_DO_SACK(tp) && tp->snd_nxt != tp->snd_max &&
!IN_FASTRECOVERY(tp)) {
tcp_seq old_snd_nxt = tp->snd_nxt;
tcp_sack_skip_sacked(&tp->scb, &tp->snd_nxt);
nsacked += tp->snd_nxt - old_snd_nxt;
}
sendalot = FALSE;
off = tp->snd_nxt - tp->snd_una;
sendwin = min(tp->snd_wnd, tp->snd_cwnd + nsacked);
sendwin = min(sendwin, tp->snd_bwnd);
flags = tcp_outflags[tp->t_state];
/*
* Get standard flags, and add SYN or FIN if requested by 'hidden'
* state flags.
*/
if (tp->t_flags & TF_NEEDFIN)
flags |= TH_FIN;
if (tp->t_flags & TF_NEEDSYN)
flags |= TH_SYN;
/*
* If in persist timeout with window of 0, send 1 byte.
* Otherwise, if window is small but nonzero
* and timer expired, we will send what we can
* and go to transmit state.
*/
if (tp->t_flags & TF_FORCE) {
if (sendwin == 0) {
/*
* If we still have some data to send, then
* clear the FIN bit. Usually this would
* happen below when it realizes that we
* aren't sending all the data. However,
* if we have exactly 1 byte of unsent data,
* then it won't clear the FIN bit below,
* and if we are in persist state, we wind
* up sending the packet without recording
* that we sent the FIN bit.
*
* We can't just blindly clear the FIN bit,
* because if we don't have any more data
* to send then the probe will be the FIN
* itself.
*/
if (off < so->so_snd.ssb_cc)
flags &= ~TH_FIN;
sendwin = 1;
} else {
tcp_callout_stop(tp, tp->tt_persist);
tp->t_rxtshift = 0;
}
}
/*
* If snd_nxt == snd_max and we have transmitted a FIN, the
* offset will be > 0 even if so_snd.ssb_cc is 0, resulting in
* a negative length. This can also occur when TCP opens up
* its congestion window while receiving additional duplicate
* acks after fast-retransmit because TCP will reset snd_nxt
* to snd_max after the fast-retransmit.
*
* In the normal retransmit-FIN-only case, however, snd_nxt will
* be set to snd_una, the offset will be 0, and the length may
* wind up 0.
*/
len = (long)ulmin(so->so_snd.ssb_cc, sendwin) - off;
/*
* Lop off SYN bit if it has already been sent. However, if this
* is SYN-SENT state and if segment contains data, suppress sending
* segment (sending the segment would be an option if we still
* did TAO and the remote host supported it).
*/
if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
flags &= ~TH_SYN;
off--, len++;
if (len > 0 && tp->t_state == TCPS_SYN_SENT)
return 0;
}
/*
* Be careful not to send data and/or FIN on SYN segments.
* This measure is needed to prevent interoperability problems
* with not fully conformant TCP implementations.
*/
if (flags & TH_SYN) {
len = 0;
flags &= ~TH_FIN;
}
if (len < 0) {
/*
* If FIN has been sent but not acked,
* but we haven't been called to retransmit,
* len will be < 0. Otherwise, window shrank
* after we sent into it. If window shrank to 0,
* cancel pending retransmit, pull snd_nxt back
* to (closed) window, and set the persist timer
* if it isn't already going. If the window didn't
* close completely, just wait for an ACK.
*/
len = 0;
if (sendwin == 0) {
tcp_callout_stop(tp, tp->tt_rexmt);
tp->t_rxtshift = 0;
tp->snd_nxt = tp->snd_una;
if (!tcp_callout_active(tp, tp->tt_persist))
tcp_setpersist(tp);
}
}
KASSERT(len >= 0, ("%s: len < 0", __func__));
/*
* Automatic sizing of send socket buffer. Often the send buffer
* size is not optimally adjusted to the actual network conditions
* at hand (delay bandwidth product). Setting the buffer size too
* small limits throughput on links with high bandwidth and high
* delay (eg. trans-continental/oceanic links). Setting the
* buffer size too big consumes too much real kernel memory,
* especially with many connections on busy servers.
*
* The criteria to step up the send buffer one notch are:
* 1. receive window of remote host is larger than send buffer
* (with a fudge factor of 5/4th);
* 2. send buffer is filled to 7/8th with data (so we actually
* have data to make use of it);
* 3. send buffer fill has not hit maximal automatic size;
* 4. our send window (slow start and cogestion controlled) is
* larger than sent but unacknowledged data in send buffer.
*
* The remote host receive window scaling factor may limit the
* growing of the send buffer before it reaches its allowed
* maximum.
*
* It scales directly with slow start or congestion window
* and does at most one step per received ACK. This fast
* scaling has the drawback of growing the send buffer beyond
* what is strictly necessary to make full use of a given
* delay*bandwith product. However testing has shown this not
* to be much of an problem. At worst we are trading wasting
* of available bandwith (the non-use of it) for wasting some
* socket buffer memory.
*
* TODO: Shrink send buffer during idle periods together
* with congestion window. Requires another timer. Has to
* wait for upcoming tcp timer rewrite.
*/
if (tcp_do_autosndbuf && so->so_snd.ssb_flags & SSB_AUTOSIZE) {
if ((tp->snd_wnd / 4 * 5) >= so->so_snd.ssb_hiwat &&
so->so_snd.ssb_cc >= (so->so_snd.ssb_hiwat / 8 * 7) &&
so->so_snd.ssb_cc < tcp_autosndbuf_max &&
sendwin >= (so->so_snd.ssb_cc - (tp->snd_nxt - tp->snd_una))) {
u_long newsize;
newsize = ulmin(so->so_snd.ssb_hiwat +
tcp_autosndbuf_inc,
tcp_autosndbuf_max);
if (!ssb_reserve(&so->so_snd, newsize, so, NULL))
atomic_clear_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
if (newsize >= (TCP_MAXWIN << tp->snd_scale))
atomic_clear_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
}
}
/*
* Truncate to the maximum segment length and ensure that FIN is
* removed if the length no longer contains the last data byte.
*/
if (len > tp->t_maxseg) {
len = tp->t_maxseg;
sendalot = TRUE;
}
if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.ssb_cc))
flags &= ~TH_FIN;
recvwin = ssb_space(&so->so_rcv);
/*
* Sender silly window avoidance. We transmit under the following
* conditions when len is non-zero:
*
* - We have a full segment
* - This is the last buffer in a write()/send() and we are
* either idle or running NODELAY
* - we've timed out (e.g. persist timer)
* - we have more then 1/2 the maximum send window's worth of
* data (receiver may be limiting the window size)
* - we need to retransmit
*/
if (len) {
if (len == tp->t_maxseg)
goto send;
/*
* NOTE! on localhost connections an 'ack' from the remote
* end may occur synchronously with the output and cause
* us to flush a buffer queued with moretocome. XXX
*
* note: the len + off check is almost certainly unnecessary.
*/
if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */
(idle || (tp->t_flags & TF_NODELAY)) &&
len + off >= so->so_snd.ssb_cc &&
!(tp->t_flags & TF_NOPUSH)) {
goto send;
}
if (tp->t_flags & TF_FORCE) /* typ. timeout case */
goto send;
if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
goto send;
if (SEQ_LT(tp->snd_nxt, tp->snd_max)) /* retransmit case */
goto send;
}
/*
* Compare available window to amount of window
* known to peer (as advertised window less
* next expected input). If the difference is at least two
* max size segments, or at least 50% of the maximum possible
* window, then want to send a window update to peer.
*/
if (recvwin > 0) {
/*
* "adv" is the amount we can increase the window,
* taking into account that we are limited by
* TCP_MAXWIN << tp->rcv_scale.
*/
long adv = min(recvwin, (long)TCP_MAXWIN << tp->rcv_scale) -
(tp->rcv_adv - tp->rcv_nxt);
long hiwat;
/*
* This ack case typically occurs when the user has drained
* the TCP socket buffer sufficiently to warrent an ack
* containing a 'pure window update'... that is, an ack that
* ONLY updates the tcp window.
*
* It is unclear why we would need to do a pure window update
* past 2 segments if we are going to do one at 1/2 the high
* water mark anyway, especially since under normal conditions
* the user program will drain the socket buffer quickly.
* The 2-segment pure window update will often add a large
* number of extra, unnecessary acks to the stream.
*
* avoid_pure_win_update now defaults to 1.
*/
if (avoid_pure_win_update == 0 ||
(tp->t_flags & TF_RXRESIZED)) {
if (adv >= (long) (2 * tp->t_maxseg)) {
goto send;
}
}
hiwat = (long)(TCP_MAXWIN << tp->rcv_scale);
if (hiwat > (long)so->so_rcv.ssb_hiwat)
hiwat = (long)so->so_rcv.ssb_hiwat;
if (adv >= hiwat / 2)
goto send;
}
/*
* Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
* is also a catch-all for the retransmit timer timeout case.
*/
if (tp->t_flags & TF_ACKNOW)
goto send;
if ((flags & TH_RST) ||
((flags & TH_SYN) && !(tp->t_flags & TF_NEEDSYN)))
goto send;
if (SEQ_GT(tp->snd_up, tp->snd_una))
goto send;
/*
* If our state indicates that FIN should be sent
* and we have not yet done so, then we need to send.
*/
if (flags & TH_FIN &&
(!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una))
goto send;
/*
* TCP window updates are not reliable, rather a polling protocol
* using ``persist'' packets is used to insure receipt of window
* updates. The three ``states'' for the output side are:
* idle not doing retransmits or persists
* persisting to move a small or zero window
* (re)transmitting and thereby not persisting
*
* tcp_callout_active(tp, tp->tt_persist)
* is true when we are in persist state.
* The TF_FORCE flag in tp->t_flags
* is set when we are called to send a persist packet.
* tcp_callout_active(tp, tp->tt_rexmt)
* is set when we are retransmitting
* The output side is idle when both timers are zero.
*
* If send window is too small, there is data to transmit, and no
* retransmit or persist is pending, then go to persist state.
* If nothing happens soon, send when timer expires:
* if window is nonzero, transmit what we can,
* otherwise force out a byte.
*/
if (so->so_snd.ssb_cc > 0 &&
!tcp_callout_active(tp, tp->tt_rexmt) &&
!tcp_callout_active(tp, tp->tt_persist)) {
tp->t_rxtshift = 0;
tcp_setpersist(tp);
}
/*
* No reason to send a segment, just return.
*/
return (0);
send:
/*
* Before ESTABLISHED, force sending of initial options
* unless TCP set not to do any options.
* NOTE: we assume that the IP/TCP header plus TCP options
* always fit in a single mbuf, leaving room for a maximum
* link header, i.e.
* max_linkhdr + sizeof(struct tcpiphdr) + optlen <= MCLBYTES
*/
optlen = 0;
if (isipv6)
hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
else
hdrlen = sizeof(struct tcpiphdr);
if (flags & TH_SYN) {
tp->snd_nxt = tp->iss;
if (!(tp->t_flags & TF_NOOPT)) {
u_short mss;
opt[0] = TCPOPT_MAXSEG;
opt[1] = TCPOLEN_MAXSEG;
mss = htons((u_short) tcp_mssopt(tp));
memcpy(opt + 2, &mss, sizeof mss);
optlen = TCPOLEN_MAXSEG;
if ((tp->t_flags & TF_REQ_SCALE) &&
(!(flags & TH_ACK) ||
(tp->t_flags & TF_RCVD_SCALE))) {
*((u_int32_t *)(opt + optlen)) = htonl(
TCPOPT_NOP << 24 |
TCPOPT_WINDOW << 16 |
TCPOLEN_WINDOW << 8 |
tp->request_r_scale);
optlen += 4;
}
if ((tcp_do_sack && !(flags & TH_ACK)) ||
tp->t_flags & TF_SACK_PERMITTED) {
uint32_t *lp = (uint32_t *)(opt + optlen);
*lp = htonl(TCPOPT_SACK_PERMITTED_ALIGNED);
optlen += TCPOLEN_SACK_PERMITTED_ALIGNED;
}
}
}
/*
* Send a timestamp and echo-reply if this is a SYN and our side
* wants to use timestamps (TF_REQ_TSTMP is set) or both our side
* and our peer have sent timestamps in our SYN's.
*/
if ((tp->t_flags & (TF_REQ_TSTMP | TF_NOOPT)) == TF_REQ_TSTMP &&
!(flags & TH_RST) &&
(!(flags & TH_ACK) || (tp->t_flags & TF_RCVD_TSTMP))) {
u_int32_t *lp = (u_int32_t *)(opt + optlen);
/* Form timestamp option as shown in appendix A of RFC 1323. */
*lp++ = htonl(TCPOPT_TSTAMP_HDR);
*lp++ = htonl(ticks);
*lp = htonl(tp->ts_recent);
optlen += TCPOLEN_TSTAMP_APPA;
}
/* Set receive buffer autosizing timestamp. */
if (tp->rfbuf_ts == 0 && (so->so_rcv.ssb_flags & SSB_AUTOSIZE))
tp->rfbuf_ts = ticks;
/*
* If this is a SACK connection and we have a block to report,
* fill in the SACK blocks in the TCP options.
*/
if ((tp->t_flags & (TF_SACK_PERMITTED | TF_NOOPT)) ==
TF_SACK_PERMITTED &&
(!LIST_EMPTY(&tp->t_segq) ||
tp->reportblk.rblk_start != tp->reportblk.rblk_end))
tcp_sack_fill_report(tp, opt, &optlen);
#ifdef TCP_SIGNATURE
if (tp->t_flags & TF_SIGNATURE) {
int i;
u_char *bp;
/*
* Initialize TCP-MD5 option (RFC2385)
*/
bp = (u_char *)opt + optlen;
*bp++ = TCPOPT_SIGNATURE;
*bp++ = TCPOLEN_SIGNATURE;
sigoff = optlen + 2;
for (i = 0; i < TCP_SIGLEN; i++)
*bp++ = 0;
optlen += TCPOLEN_SIGNATURE;
/*
* Terminate options list and maintain 32-bit alignment.
*/
*bp++ = TCPOPT_NOP;
*bp++ = TCPOPT_EOL;
optlen += 2;
}
#endif /* TCP_SIGNATURE */
KASSERT(optlen <= TCP_MAXOLEN, ("too many TCP options"));
hdrlen += optlen;
if (isipv6) {
ipoptlen = ip6_optlen(inp);
} else {
if (inp->inp_options) {
ipoptlen = inp->inp_options->m_len -
offsetof(struct ipoption, ipopt_list);
} else {
/*
* Tcp output routine: figure out what should be sent and send it.
*/
int
tcp_output(struct tcpcb *tp)
{
struct inpcb * const inp = tp->t_inpcb;
struct socket *so = inp->inp_socket;
long len, recvwin, sendwin;
int nsacked = 0;
int off, flags, error = 0;
#ifdef TCP_SIGNATURE
int sigoff = 0;
#endif
struct mbuf *m;
struct ip *ip;
struct tcphdr *th;
u_char opt[TCP_MAXOLEN];
unsigned int ipoptlen, optlen, hdrlen;
int idle;
boolean_t sendalot;
struct ip6_hdr *ip6;
#ifdef INET6
const boolean_t isipv6 = INP_ISIPV6(inp);
#else
const boolean_t isipv6 = FALSE;
#endif
boolean_t can_tso = FALSE, use_tso;
boolean_t report_sack, idle_cwv = FALSE;
u_int segsz, tso_hlen, tso_lenmax = 0;
int segcnt = 0;
boolean_t need_sched = FALSE;
KKASSERT(so->so_port == &curthread->td_msgport);
/*
* Determine length of data that should be transmitted,
* and flags that will be used.
* If there is some data or critical controls (SYN, RST)
* to send, then transmit; otherwise, investigate further.
*/
/*
* If we have been idle for a while, the send congestion window
* could be no longer representative of the current state of the
* link; need to validate congestion window. However, we should
* not perform congestion window validation here, since we could
* be asked to send pure ACK.
*/
if (tp->snd_max == tp->snd_una &&
(ticks - tp->snd_last) >= tp->t_rxtcur && tcp_idle_restart)
idle_cwv = TRUE;
/*
* Calculate whether the transmit stream was previously idle
* and adjust TF_LASTIDLE for the next time.
*/
idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
if (idle && (tp->t_flags & TF_MORETOCOME))
tp->t_flags |= TF_LASTIDLE;
else
tp->t_flags &= ~TF_LASTIDLE;
if (TCP_DO_SACK(tp) && tp->snd_nxt != tp->snd_max &&
!IN_FASTRECOVERY(tp))
nsacked = tcp_sack_bytes_below(&tp->scb, tp->snd_nxt);
/*
* Find out whether TSO could be used or not
*
* For TSO capable devices, the following assumptions apply to
* the processing of TCP flags:
* - If FIN is set on the large TCP segment, the device must set
* FIN on the last segment that it creates from the large TCP
* segment.
* - If PUSH is set on the large TCP segment, the device must set
* PUSH on the last segment that it creates from the large TCP
* segment.
*/
#if !defined(IPSEC) && !defined(FAST_IPSEC)
if (tcp_do_tso
#ifdef TCP_SIGNATURE
&& (tp->t_flags & TF_SIGNATURE) == 0
#endif
) {
if (!isipv6) {
struct rtentry *rt = inp->inp_route.ro_rt;
if (rt != NULL && (rt->rt_flags & RTF_UP) &&
(rt->rt_ifp->if_hwassist & CSUM_TSO)) {
can_tso = TRUE;
tso_lenmax = rt->rt_ifp->if_tsolen;
}
}
}
#endif /* !IPSEC && !FAST_IPSEC */
again:
m = NULL;
ip = NULL;
th = NULL;
ip6 = NULL;
if ((tp->t_flags & (TF_SACK_PERMITTED | TF_NOOPT)) ==
TF_SACK_PERMITTED &&
(!TAILQ_EMPTY(&tp->t_segq) ||
tp->reportblk.rblk_start != tp->reportblk.rblk_end))
report_sack = TRUE;
else
report_sack = FALSE;
/* Make use of SACK information when slow-starting after a RTO. */
if (TCP_DO_SACK(tp) && tp->snd_nxt != tp->snd_max &&
!IN_FASTRECOVERY(tp)) {
tcp_seq old_snd_nxt = tp->snd_nxt;
tcp_sack_skip_sacked(&tp->scb, &tp->snd_nxt);
nsacked += tp->snd_nxt - old_snd_nxt;
}
sendalot = FALSE;
off = tp->snd_nxt - tp->snd_una;
sendwin = min(tp->snd_wnd, tp->snd_cwnd + nsacked);
sendwin = min(sendwin, tp->snd_bwnd);
flags = tcp_outflags[tp->t_state];
/*
* Get standard flags, and add SYN or FIN if requested by 'hidden'
* state flags.
*/
if (tp->t_flags & TF_NEEDFIN)
flags |= TH_FIN;
if (tp->t_flags & TF_NEEDSYN)
flags |= TH_SYN;
/*
* If in persist timeout with window of 0, send 1 byte.
* Otherwise, if window is small but nonzero
* and timer expired, we will send what we can
* and go to transmit state.
*/
if (tp->t_flags & TF_FORCE) {
if (sendwin == 0) {
/*
* If we still have some data to send, then
* clear the FIN bit. Usually this would
* happen below when it realizes that we
* aren't sending all the data. However,
* if we have exactly 1 byte of unsent data,
* then it won't clear the FIN bit below,
* and if we are in persist state, we wind
* up sending the packet without recording
* that we sent the FIN bit.
*
* We can't just blindly clear the FIN bit,
* because if we don't have any more data
* to send then the probe will be the FIN
* itself.
*/
if (off < so->so_snd.ssb_cc)
flags &= ~TH_FIN;
sendwin = 1;
} else {
tcp_callout_stop(tp, tp->tt_persist);
tp->t_rxtshift = 0;
}
}
/*
* If snd_nxt == snd_max and we have transmitted a FIN, the
* offset will be > 0 even if so_snd.ssb_cc is 0, resulting in
* a negative length. This can also occur when TCP opens up
* its congestion window while receiving additional duplicate
* acks after fast-retransmit because TCP will reset snd_nxt
* to snd_max after the fast-retransmit.
*
* A negative length can also occur when we are in the
* TCPS_SYN_RECEIVED state due to a simultanious connect where
* our SYN has not been acked yet.
*
* In the normal retransmit-FIN-only case, however, snd_nxt will
* be set to snd_una, the offset will be 0, and the length may
* wind up 0.
*/
len = (long)ulmin(so->so_snd.ssb_cc, sendwin) - off;
/*
* Lop off SYN bit if it has already been sent. However, if this
* is SYN-SENT state and if segment contains data, suppress sending
* segment (sending the segment would be an option if we still
* did TAO and the remote host supported it).
*/
if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
flags &= ~TH_SYN;
off--, len++;
if (len > 0 && tp->t_state == TCPS_SYN_SENT) {
tp->t_flags &= ~(TF_ACKNOW | TF_XMITNOW);
return 0;
}
}
/*
* Be careful not to send data and/or FIN on SYN segments.
* This measure is needed to prevent interoperability problems
* with not fully conformant TCP implementations.
*/
if (flags & TH_SYN) {
len = 0;
flags &= ~TH_FIN;
}
if (len < 0) {
/*
* A negative len can occur if our FIN has been sent but not
* acked, or if we are in a simultanious connect in the
* TCPS_SYN_RECEIVED state with our SYN sent but not yet
* acked.
*
* If our window has contracted to 0 in the FIN case
* (which can only occur if we have NOT been called to
* retransmit as per code a few paragraphs up) then we
* want to shift the retransmit timer over to the
* persist timer.
*
* However, if we are in the TCPS_SYN_RECEIVED state
* (the SYN case) we will be in a simultanious connect and
* the window may be zero degeneratively. In this case we
* do not want to shift to the persist timer after the SYN
* or the SYN+ACK transmission.
*/
len = 0;
if (sendwin == 0 && tp->t_state != TCPS_SYN_RECEIVED) {
tcp_callout_stop(tp, tp->tt_rexmt);
tp->t_rxtshift = 0;
tp->snd_nxt = tp->snd_una;
if (!tcp_callout_active(tp, tp->tt_persist))
tcp_setpersist(tp);
}
}
KASSERT(len >= 0, ("%s: len < 0", __func__));
/*
* Automatic sizing of send socket buffer. Often the send buffer
* size is not optimally adjusted to the actual network conditions
* at hand (delay bandwidth product). Setting the buffer size too
* small limits throughput on links with high bandwidth and high
* delay (eg. trans-continental/oceanic links). Setting the
* buffer size too big consumes too much real kernel memory,
* especially with many connections on busy servers.
*
* The criteria to step up the send buffer one notch are:
* 1. receive window of remote host is larger than send buffer
* (with a fudge factor of 5/4th);
* 2. hiwat has not significantly exceeded bwnd (inflight)
* (bwnd is a maximal value if inflight is disabled).
* 3. send buffer is filled to 7/8th with data (so we actually
* have data to make use of it);
* 4. hiwat has not hit maximal automatic size;
* 5. our send window (slow start and cogestion controlled) is
* larger than sent but unacknowledged data in send buffer.
*
* The remote host receive window scaling factor may limit the
* growing of the send buffer before it reaches its allowed
* maximum.
*
* It scales directly with slow start or congestion window
* and does at most one step per received ACK. This fast
* scaling has the drawback of growing the send buffer beyond
* what is strictly necessary to make full use of a given
* delay*bandwith product. However testing has shown this not
* to be much of an problem. At worst we are trading wasting
* of available bandwith (the non-use of it) for wasting some
* socket buffer memory.
*
* The criteria for shrinking the buffer is based solely on
* the inflight code (snd_bwnd). If inflight is disabled,
* the buffer will not be shrinked. Note that snd_bwnd already
* has a fudge factor. Our test adds a little hysteresis.
*/
if (tcp_do_autosndbuf && (so->so_snd.ssb_flags & SSB_AUTOSIZE)) {
const int asbinc = tcp_autosndbuf_inc;
const int hiwat = so->so_snd.ssb_hiwat;
const int lowat = so->so_snd.ssb_lowat;
u_long newsize;
if ((tp->snd_wnd / 4 * 5) >= hiwat &&
so->so_snd.ssb_cc >= (hiwat / 8 * 7) &&
hiwat < tp->snd_bwnd + hiwat / 10 &&
hiwat + asbinc < tcp_autosndbuf_max &&
hiwat < (TCP_MAXWIN << tp->snd_scale) &&
sendwin >= (so->so_snd.ssb_cc -
(tp->snd_nxt - tp->snd_una))) {
newsize = ulmin(hiwat + asbinc, tcp_autosndbuf_max);
if (!ssb_reserve(&so->so_snd, newsize, so, NULL))
atomic_clear_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
#if 0
if (newsize >= (TCP_MAXWIN << tp->snd_scale))
atomic_clear_int(&so->so_snd.ssb_flags, SSB_AUTOSIZE);
#endif
} else if ((long)tp->snd_bwnd <
(long)(hiwat * 3 / 4 - lowat - asbinc) &&
hiwat > tp->t_maxseg * 2 + asbinc &&
hiwat + asbinc >= tcp_autosndbuf_min &&
tcp_do_autosndbuf == 1) {
newsize = ulmax(hiwat - asbinc, tp->t_maxseg * 2);
ssb_reserve(&so->so_snd, newsize, so, NULL);
}
}
/*
* Don't use TSO, if:
* - Congestion window needs validation
* - There are SACK blocks to report
* - RST or SYN flags is set
* - URG will be set
*
* XXX
* Checking for SYN|RST looks overkill, just to be safe than sorry
*/
use_tso = can_tso;
if (report_sack || idle_cwv || (flags & (TH_RST | TH_SYN)))
use_tso = FALSE;
if (use_tso) {
tcp_seq ugr_nxt = tp->snd_nxt;
if ((flags & TH_FIN) && (tp->t_flags & TF_SENTFIN) &&
tp->snd_nxt == tp->snd_max)
--ugr_nxt;
if (SEQ_GT(tp->snd_up, ugr_nxt))
use_tso = FALSE;
}
if (use_tso) {
/*
* Find out segment size and header length for TSO
*/
error = tcp_tso_getsize(tp, &segsz, &tso_hlen);
if (error)
use_tso = FALSE;
}
if (!use_tso) {
segsz = tp->t_maxseg;
tso_hlen = 0; /* not used */
}
/*
* Truncate to the maximum segment length if not TSO, and ensure that
* FIN is removed if the length no longer contains the last data byte.
*/
if (len > segsz) {
if (!use_tso) {
len = segsz;
++segcnt;
} else {
int nsegs;
if (__predict_false(tso_lenmax < segsz))
tso_lenmax = segsz << 1;
/*
* Truncate TSO transfers to (IP_MAXPACKET - iphlen -
* thoff), and make sure that we send equal size
* transfers down the stack (rather than big-small-
* big-small-...).
*/
len = min(len, tso_lenmax);
nsegs = min(len, (IP_MAXPACKET - tso_hlen)) / segsz;
KKASSERT(nsegs > 0);
len = nsegs * segsz;
if (len <= segsz) {
use_tso = FALSE;
++segcnt;
} else {
segcnt += nsegs;
}
}
sendalot = TRUE;
} else {
use_tso = FALSE;
if (len > 0)
++segcnt;
}
if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.ssb_cc))
flags &= ~TH_FIN;
recvwin = ssb_space(&so->so_rcv);
/*
* Sender silly window avoidance. We transmit under the following
* conditions when len is non-zero:
*
* - We have a full segment
* - This is the last buffer in a write()/send() and we are
* either idle or running NODELAY
* - we've timed out (e.g. persist timer)
* - we have more then 1/2 the maximum send window's worth of
* data (receiver may be limiting the window size)
* - we need to retransmit
*/
if (len) {
if (len >= segsz)
goto send;
/*
* NOTE! on localhost connections an 'ack' from the remote
* end may occur synchronously with the output and cause
* us to flush a buffer queued with moretocome. XXX
*
* note: the len + off check is almost certainly unnecessary.
*/
if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */
(idle || (tp->t_flags & TF_NODELAY)) &&
len + off >= so->so_snd.ssb_cc &&
!(tp->t_flags & TF_NOPUSH)) {
goto send;
}
if (tp->t_flags & TF_FORCE) /* typ. timeout case */
goto send;
if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
goto send;
if (SEQ_LT(tp->snd_nxt, tp->snd_max)) /* retransmit case */
goto send;
if (tp->t_flags & TF_XMITNOW)
goto send;
}
/*
* Compare available window to amount of window
* known to peer (as advertised window less
* next expected input). If the difference is at least two
* max size segments, or at least 50% of the maximum possible
* window, then want to send a window update to peer.
*/
if (recvwin > 0) {
/*
* "adv" is the amount we can increase the window,
* taking into account that we are limited by
* TCP_MAXWIN << tp->rcv_scale.
*/
long adv = min(recvwin, (long)TCP_MAXWIN << tp->rcv_scale) -
(tp->rcv_adv - tp->rcv_nxt);
long hiwat;
/*
* This ack case typically occurs when the user has drained
* the TCP socket buffer sufficiently to warrent an ack
* containing a 'pure window update'... that is, an ack that
* ONLY updates the tcp window.
*
* It is unclear why we would need to do a pure window update
* past 2 segments if we are going to do one at 1/2 the high
* water mark anyway, especially since under normal conditions
* the user program will drain the socket buffer quickly.
* The 2-segment pure window update will often add a large
* number of extra, unnecessary acks to the stream.
*
* avoid_pure_win_update now defaults to 1.
*/
if (avoid_pure_win_update == 0 ||
(tp->t_flags & TF_RXRESIZED)) {
if (adv >= (long) (2 * segsz)) {
goto send;
}
}
hiwat = (long)(TCP_MAXWIN << tp->rcv_scale);
if (hiwat > (long)so->so_rcv.ssb_hiwat)
hiwat = (long)so->so_rcv.ssb_hiwat;
if (adv >= hiwat / 2)
goto send;
}
/*
* Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
* is also a catch-all for the retransmit timer timeout case.
*/
if (tp->t_flags & TF_ACKNOW)
goto send;
if ((flags & TH_RST) ||
((flags & TH_SYN) && !(tp->t_flags & TF_NEEDSYN)))
goto send;
if (SEQ_GT(tp->snd_up, tp->snd_una))
goto send;
/*
* If our state indicates that FIN should be sent
* and we have not yet done so, then we need to send.
*/
if ((flags & TH_FIN) &&
(!(tp->t_flags & TF_SENTFIN) || tp->snd_nxt == tp->snd_una))
goto send;
/*
* TCP window updates are not reliable, rather a polling protocol
* using ``persist'' packets is used to insure receipt of window
* updates. The three ``states'' for the output side are:
* idle not doing retransmits or persists
* persisting to move a small or zero window
* (re)transmitting and thereby not persisting
*
* tcp_callout_active(tp, tp->tt_persist)
* is true when we are in persist state.
* The TF_FORCE flag in tp->t_flags
* is set when we are called to send a persist packet.
* tcp_callout_active(tp, tp->tt_rexmt)
* is set when we are retransmitting
* The output side is idle when both timers are zero.
*
* If send window is too small, there is data to transmit, and no
* retransmit or persist is pending, then go to persist state.
*
* If nothing happens soon, send when timer expires:
* if window is nonzero, transmit what we can, otherwise force out
* a byte.
*
* Don't try to set the persist state if we are in TCPS_SYN_RECEIVED
* with data pending. This situation can occur during a
* simultanious connect.
*/
if (so->so_snd.ssb_cc > 0 &&
tp->t_state != TCPS_SYN_RECEIVED &&
!tcp_callout_active(tp, tp->tt_rexmt) &&
!tcp_callout_active(tp, tp->tt_persist)) {
tp->t_rxtshift = 0;
tcp_setpersist(tp);
}
/*
* No reason to send a segment, just return.
*/
tp->t_flags &= ~TF_XMITNOW;
return (0);
send:
if (need_sched && len > 0) {
tcp_output_sched(tp);
return 0;
}
/*
* Before ESTABLISHED, force sending of initial options
* unless TCP set not to do any options.
* NOTE: we assume that the IP/TCP header plus TCP options
* always fit in a single mbuf, leaving room for a maximum
* link header, i.e.
* max_linkhdr + sizeof(struct tcpiphdr) + optlen <= MCLBYTES
*/
optlen = 0;
if (isipv6)
hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
else
hdrlen = sizeof(struct tcpiphdr);
if (flags & TH_SYN) {
tp->snd_nxt = tp->iss;
if (!(tp->t_flags & TF_NOOPT)) {
u_short mss;
opt[0] = TCPOPT_MAXSEG;
opt[1] = TCPOLEN_MAXSEG;
mss = htons((u_short) tcp_mssopt(tp));
memcpy(opt + 2, &mss, sizeof mss);
optlen = TCPOLEN_MAXSEG;
if ((tp->t_flags & TF_REQ_SCALE) &&
(!(flags & TH_ACK) ||
(tp->t_flags & TF_RCVD_SCALE))) {
*((u_int32_t *)(opt + optlen)) = htonl(
TCPOPT_NOP << 24 |
TCPOPT_WINDOW << 16 |
TCPOLEN_WINDOW << 8 |
tp->request_r_scale);
optlen += 4;
}
if ((tcp_do_sack && !(flags & TH_ACK)) ||
tp->t_flags & TF_SACK_PERMITTED) {
uint32_t *lp = (uint32_t *)(opt + optlen);
*lp = htonl(TCPOPT_SACK_PERMITTED_ALIGNED);
optlen += TCPOLEN_SACK_PERMITTED_ALIGNED;
}
}
}
/*
* Send a timestamp and echo-reply if this is a SYN and our side
* wants to use timestamps (TF_REQ_TSTMP is set) or both our side
* and our peer have sent timestamps in our SYN's.
*/
if ((tp->t_flags & (TF_REQ_TSTMP | TF_NOOPT)) == TF_REQ_TSTMP &&
!(flags & TH_RST) &&
(!(flags & TH_ACK) || (tp->t_flags & TF_RCVD_TSTMP))) {
u_int32_t *lp = (u_int32_t *)(opt + optlen);
/* Form timestamp option as shown in appendix A of RFC 1323. */
*lp++ = htonl(TCPOPT_TSTAMP_HDR);
*lp++ = htonl(ticks);
*lp = htonl(tp->ts_recent);
optlen += TCPOLEN_TSTAMP_APPA;
}
/* Set receive buffer autosizing timestamp. */
if (tp->rfbuf_ts == 0 && (so->so_rcv.ssb_flags & SSB_AUTOSIZE))
tp->rfbuf_ts = ticks;
/*
* If this is a SACK connection and we have a block to report,
* fill in the SACK blocks in the TCP options.
*/
if (report_sack)
tcp_sack_fill_report(tp, opt, &optlen);
#ifdef TCP_SIGNATURE
if (tp->t_flags & TF_SIGNATURE) {
int i;
u_char *bp;
/*
* Initialize TCP-MD5 option (RFC2385)
*/
bp = (u_char *)opt + optlen;
*bp++ = TCPOPT_SIGNATURE;
*bp++ = TCPOLEN_SIGNATURE;
sigoff = optlen + 2;
for (i = 0; i < TCP_SIGLEN; i++)
*bp++ = 0;
optlen += TCPOLEN_SIGNATURE;
/*
* Terminate options list and maintain 32-bit alignment.
*/
*bp++ = TCPOPT_NOP;
*bp++ = TCPOPT_EOL;
optlen += 2;
}
#endif /* TCP_SIGNATURE */
KASSERT(optlen <= TCP_MAXOLEN, ("too many TCP options"));
hdrlen += optlen;
if (isipv6) {
ipoptlen = ip6_optlen(inp);
} else {
if (inp->inp_options) {
ipoptlen = inp->inp_options->m_len -
offsetof(struct ipoption, ipopt_list);
} else {
static
#ifndef GPROF
inline
#endif
int
tcp_segsize(struct tcpcb *tp, int *txsegsizep, int *rxsegsizep,
bool *alwaysfragp)
{
#ifdef INET
struct inpcb *inp = tp->t_inpcb;
#endif
#ifdef INET6
struct in6pcb *in6p = tp->t_in6pcb;
#endif
struct socket *so = NULL;
struct rtentry *rt;
struct ifnet *ifp;
int size;
int hdrlen;
int optlen;
*alwaysfragp = false;
#ifdef DIAGNOSTIC
if (tp->t_inpcb && tp->t_in6pcb)
panic("tcp_segsize: both t_inpcb and t_in6pcb are set");
#endif
switch (tp->t_family) {
#ifdef INET
case AF_INET:
hdrlen = sizeof(struct ip) + sizeof(struct tcphdr);
break;
#endif
#ifdef INET6
case AF_INET6:
hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
break;
#endif
default:
size = tcp_mssdflt;
goto out;
}
rt = NULL;
#ifdef INET
if (inp) {
rt = in_pcbrtentry(inp);
so = inp->inp_socket;
}
#endif
#ifdef INET6
if (in6p) {
rt = in6_pcbrtentry(in6p);
so = in6p->in6p_socket;
}
#endif
if (rt == NULL) {
size = tcp_mssdflt;
goto out;
}
ifp = rt->rt_ifp;
size = tcp_mssdflt;
if (tp->t_mtudisc && rt->rt_rmx.rmx_mtu != 0) {
#ifdef INET6
if (in6p && rt->rt_rmx.rmx_mtu < IPV6_MMTU) {
/*
* RFC2460 section 5, last paragraph: if path MTU is
* smaller than 1280, use 1280 as packet size and
* attach fragment header.
*/
size = IPV6_MMTU - hdrlen - sizeof(struct ip6_frag);
*alwaysfragp = true;
} else
size = rt->rt_rmx.rmx_mtu - hdrlen;
#else
size = rt->rt_rmx.rmx_mtu - hdrlen;
#endif
} else if (ifp->if_flags & IFF_LOOPBACK)
size = ifp->if_mtu - hdrlen;
#ifdef INET
else if (inp && tp->t_mtudisc)
size = ifp->if_mtu - hdrlen;
else if (inp && in_localaddr(inp->inp_faddr))
size = ifp->if_mtu - hdrlen;
#endif
#ifdef INET6
else if (in6p) {
#ifdef INET
if (IN6_IS_ADDR_V4MAPPED(&in6p->in6p_faddr)) {
/* mapped addr case */
struct in_addr d;
bcopy(&in6p->in6p_faddr.s6_addr32[3], &d, sizeof(d));
if (tp->t_mtudisc || in_localaddr(d))
size = ifp->if_mtu - hdrlen;
} else
#endif
{
/*
* for IPv6, path MTU discovery is always turned on,
* or the node must use packet size <= 1280.
*/
size = tp->t_mtudisc ? IN6_LINKMTU(ifp) : IPV6_MMTU;
size -= hdrlen;
}
}
#endif
out:
/*
* Now we must make room for whatever extra TCP/IP options are in
* the packet.
*/
optlen = tcp_optlen(tp);
/*
* XXX tp->t_ourmss should have the right size, but without this code
* fragmentation will occur... need more investigation
*/
#ifdef INET
if (inp) {
#if defined(IPSEC)
if (ipsec_used &&
!IPSEC_PCB_SKIP_IPSEC(inp->inp_sp, IPSEC_DIR_OUTBOUND))
optlen += ipsec4_hdrsiz_tcp(tp);
#endif
optlen += ip_optlen(inp);
}
#endif
#ifdef INET6
#ifdef INET
if (in6p && tp->t_family == AF_INET) {
#if defined(IPSEC)
if (ipsec_used &&
!IPSEC_PCB_SKIP_IPSEC(in6p->in6p_sp, IPSEC_DIR_OUTBOUND))
optlen += ipsec4_hdrsiz_tcp(tp);
#endif
/* XXX size -= ip_optlen(in6p); */
} else
#endif
if (in6p && tp->t_family == AF_INET6) {
#if defined(IPSEC)
if (ipsec_used &&
!IPSEC_PCB_SKIP_IPSEC(in6p->in6p_sp, IPSEC_DIR_OUTBOUND))
optlen += ipsec6_hdrsiz_tcp(tp);
#endif
optlen += ip6_optlen(in6p);
}
#endif
size -= optlen;
/* there may not be any room for data if mtu is too small */
if (size < 0)
return (EMSGSIZE);
/*
* *rxsegsizep holds *estimated* inbound segment size (estimation
* assumes that path MTU is the same for both ways). this is only
* for silly window avoidance, do not use the value for other purposes.
*
* ipseclen is subtracted from both sides, this may not be right.
* I'm not quite sure about this (could someone comment).
*/
*txsegsizep = min(tp->t_peermss - optlen, size);
/*
* Never send more than half a buffer full. This insures that we can
* always keep 2 packets on the wire, no matter what SO_SNDBUF is, and
* therefore acks will never be delayed unless we run out of data to
* transmit.
*/
if (so)
*txsegsizep = min(so->so_snd.sb_hiwat >> 1, *txsegsizep);
*rxsegsizep = min(tp->t_ourmss - optlen, size);
if (*txsegsizep != tp->t_segsz) {
/*
* If the new segment size is larger, we don't want to
* mess up the congestion window, but if it is smaller
* we'll have to reduce the congestion window to ensure
* that we don't get into trouble with initial windows
* and the rest. In any case, if the segment size
* has changed, chances are the path has, too, and
* our congestion window will be different.
*/
if (*txsegsizep < tp->t_segsz) {
tp->snd_cwnd = max((tp->snd_cwnd / tp->t_segsz)
* *txsegsizep, *txsegsizep);
tp->snd_ssthresh = max((tp->snd_ssthresh / tp->t_segsz)
* *txsegsizep, *txsegsizep);
}
tp->t_segsz = *txsegsizep;
}
return (0);
}
/*
* Tcp output routine: figure out what should be sent and send it.
*/
int
tcp_output(struct tcpcb *tp)
{
struct socket *so = tp->t_inpcb->inp_socket;
long len, recwin, sendwin;
int off, flags, error;
#ifdef TCP_SIGNATURE
int sigoff = 0;
#endif
struct mbuf *m;
struct ip *ip = NULL;
struct ipovly *ipov = NULL;
struct tcphdr *th;
u_char opt[TCP_MAXOLEN];
unsigned ipoptlen, optlen, hdrlen;
int idle, sendalot;
int i, sack_rxmit;
int sack_bytes_rxmt;
struct sackhole *p;
#if 0
int maxburst = TCP_MAXBURST;
#endif
struct rmxp_tao tao;
#ifdef INET6
struct ip6_hdr *ip6 = NULL;
int isipv6;
bzero(&tao, sizeof(tao));
isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV6) != 0;
#endif
#ifdef TCP_ECN
int needect;
#endif
INP_LOCK_ASSERT(tp->t_inpcb);
/*
* Determine length of data that should be transmitted,
* and flags that will be used.
* If there is some data or critical controls (SYN, RST)
* to send, then transmit; otherwise, investigate further.
*/
idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
if (idle && (ticks - tp->t_rcvtime) >= tp->t_rxtcur) {
/*
* We have been idle for "a while" and no acks are
* expected to clock out any data we send --
* slow start to get ack "clock" running again.
*
* Set the slow-start flight size depending on whether
* this is a local network or not.
*/
int ss = ss_fltsz;
#ifdef INET6
if (isipv6) {
if (in6_localaddr(&tp->t_inpcb->in6p_faddr))
ss = ss_fltsz_local;
} else
#endif
if (in_localaddr(tp->t_inpcb->inp_faddr))
ss = ss_fltsz_local;
tp->snd_cwnd = tp->t_maxseg * ss;
}
tp->t_flags &= ~TF_LASTIDLE;
if (idle) {
if (tp->t_flags & TF_MORETOCOME) {
tp->t_flags |= TF_LASTIDLE;
idle = 0;
}
}
again:
/*
* If we've recently taken a timeout, snd_max will be greater than
* snd_nxt. There may be SACK information that allows us to avoid
* resending already delivered data. Adjust snd_nxt accordingly.
*/
if (tp->sack_enable && SEQ_LT(tp->snd_nxt, tp->snd_max))
tcp_sack_adjust(tp);
sendalot = 0;
off = tp->snd_nxt - tp->snd_una;
sendwin = min(tp->snd_wnd, tp->snd_cwnd);
sendwin = min(sendwin, tp->snd_bwnd);
flags = tcp_outflags[tp->t_state];
/*
* Send any SACK-generated retransmissions. If we're explicitly trying
* to send out new data (when sendalot is 1), bypass this function.
* If we retransmit in fast recovery mode, decrement snd_cwnd, since
* we're replacing a (future) new transmission with a retransmission
* now, and we previously incremented snd_cwnd in tcp_input().
*/
/*
* Still in sack recovery , reset rxmit flag to zero.
*/
sack_rxmit = 0;
sack_bytes_rxmt = 0;
len = 0;
p = NULL;
if (tp->sack_enable && IN_FASTRECOVERY(tp) &&
(p = tcp_sack_output(tp, &sack_bytes_rxmt))) {
long cwin;
cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt;
if (cwin < 0)
cwin = 0;
/* Do not retransmit SACK segments beyond snd_recover */
if (SEQ_GT(p->end, tp->snd_recover)) {
/*
* (At least) part of sack hole extends beyond
* snd_recover. Check to see if we can rexmit data
* for this hole.
*/
if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
/*
* Can't rexmit any more data for this hole.
* That data will be rexmitted in the next
* sack recovery episode, when snd_recover
* moves past p->rxmit.
*/
p = NULL;
goto after_sack_rexmit;
} else
/* Can rexmit part of the current hole */
len = ((long)ulmin(cwin,
tp->snd_recover - p->rxmit));
} else
len = ((long)ulmin(cwin, p->end - p->rxmit));
off = p->rxmit - tp->snd_una;
KASSERT(off >= 0,("%s: sack block to the left of una : %d",
__func__, off));
if (len > 0) {
sack_rxmit = 1;
sendalot = 1;
tcpstat.tcps_sack_rexmits++;
tcpstat.tcps_sack_rexmit_bytes +=
min(len, tp->t_maxseg);
}
}
after_sack_rexmit:
/*
* Get standard flags, and add SYN or FIN if requested by 'hidden'
* state flags.
*/
if (tp->t_flags & TF_NEEDFIN)
flags |= TH_FIN;
if (tp->t_flags & TF_NEEDSYN)
flags |= TH_SYN;
SOCKBUF_LOCK(&so->so_snd);
/*
* If in persist timeout with window of 0, send 1 byte.
* Otherwise, if window is small but nonzero
* and timer expired, we will send what we can
* and go to transmit state.
*/
if (tp->t_force) {
if (sendwin == 0) {
/*
* If we still have some data to send, then
* clear the FIN bit. Usually this would
* happen below when it realizes that we
* aren't sending all the data. However,
* if we have exactly 1 byte of unsent data,
* then it won't clear the FIN bit below,
* and if we are in persist state, we wind
* up sending the packet without recording
* that we sent the FIN bit.
*
* We can't just blindly clear the FIN bit,
* because if we don't have any more data
* to send then the probe will be the FIN
* itself.
*/
if (off < so->so_snd.sb_cc)
flags &= ~TH_FIN;
sendwin = 1;
} else {
callout_stop(tp->tt_persist);
tp->t_rxtshift = 0;
}
}
/*
* If snd_nxt == snd_max and we have transmitted a FIN, the
* offset will be > 0 even if so_snd.sb_cc is 0, resulting in
* a negative length. This can also occur when TCP opens up
* its congestion window while receiving additional duplicate
* acks after fast-retransmit because TCP will reset snd_nxt
* to snd_max after the fast-retransmit.
*
* In the normal retransmit-FIN-only case, however, snd_nxt will
* be set to snd_una, the offset will be 0, and the length may
* wind up 0.
*
* If sack_rxmit is true we are retransmitting from the scoreboard
* in which case len is already set.
*/
if (sack_rxmit == 0) {
if (sack_bytes_rxmt == 0)
len = ((long)ulmin(so->so_snd.sb_cc, sendwin) - off);
else {
long cwin;
/*
* We are inside of a SACK recovery episode and are
* sending new data, having retransmitted all the
* data possible in the scoreboard.
*/
len = ((long)ulmin(so->so_snd.sb_cc, tp->snd_wnd)
- off);
/*
* Don't remove this (len > 0) check !
* We explicitly check for len > 0 here (although it
* isn't really necessary), to work around a gcc
* optimization issue - to force gcc to compute
* len above. Without this check, the computation
* of len is bungled by the optimizer.
*/
if (len > 0) {
cwin = tp->snd_cwnd -
(tp->snd_nxt - tp->sack_newdata) -
sack_bytes_rxmt;
if (cwin < 0)
cwin = 0;
len = lmin(len, cwin);
}
}
}
/*
* Lop off SYN bit if it has already been sent. However, if this
* is SYN-SENT state and if segment contains data and if we don't
* know that foreign host supports TAO, suppress sending segment.
*/
if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
flags &= ~TH_SYN;
off--, len++;
if (tcp_do_rfc1644)
tcp_hc_gettao(&tp->t_inpcb->inp_inc, &tao);
if (len > 0 && tp->t_state == TCPS_SYN_SENT &&
tao.tao_ccsent == 0)
goto just_return;
}
/*
* Be careful not to send data and/or FIN on SYN segments
* in cases when no CC option will be sent.
* This measure is needed to prevent interoperability problems
* with not fully conformant TCP implementations.
*/
if ((flags & TH_SYN) &&
((tp->t_flags & TF_NOOPT) || !(tp->t_flags & TF_REQ_CC) ||
((flags & TH_ACK) && !(tp->t_flags & TF_RCVD_CC)))) {
len = 0;
flags &= ~TH_FIN;
}
if (len < 0) {
/*
* If FIN has been sent but not acked,
* but we haven't been called to retransmit,
* len will be < 0. Otherwise, window shrank
* after we sent into it. If window shrank to 0,
* cancel pending retransmit, pull snd_nxt back
* to (closed) window, and set the persist timer
* if it isn't already going. If the window didn't
* close completely, just wait for an ACK.
*/
len = 0;
if (sendwin == 0) {
callout_stop(tp->tt_rexmt);
tp->t_rxtshift = 0;
tp->snd_nxt = tp->snd_una;
if (!callout_active(tp->tt_persist))
tcp_setpersist(tp);
}
}
/*
* len will be >= 0 after this point. Truncate to the maximum
* segment length and ensure that FIN is removed if the length
* no longer contains the last data byte.
*/
if (len > tp->t_maxseg) {
len = tp->t_maxseg;
sendalot = 1;
}
if (sack_rxmit) {
if (SEQ_LT(p->rxmit + len, tp->snd_una + so->so_snd.sb_cc))
flags &= ~TH_FIN;
} else {
if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.sb_cc))
flags &= ~TH_FIN;
}
recwin = sbspace(&so->so_rcv);
/*
* Sender silly window avoidance. We transmit under the following
* conditions when len is non-zero:
*
* - We have a full segment
* - This is the last buffer in a write()/send() and we are
* either idle or running NODELAY
* - we've timed out (e.g. persist timer)
* - we have more then 1/2 the maximum send window's worth of
* data (receiver may be limited the window size)
* - we need to retransmit
*/
if (len) {
if (len == tp->t_maxseg)
goto send;
/*
* NOTE! on localhost connections an 'ack' from the remote
* end may occur synchronously with the output and cause
* us to flush a buffer queued with moretocome. XXX
*
* note: the len + off check is almost certainly unnecessary.
*/
if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */
(idle || (tp->t_flags & TF_NODELAY)) &&
len + off >= so->so_snd.sb_cc &&
(tp->t_flags & TF_NOPUSH) == 0) {
goto send;
}
if (tp->t_force) /* typ. timeout case */
goto send;
if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
goto send;
if (SEQ_LT(tp->snd_nxt, tp->snd_max)) /* retransmit case */
goto send;
if (sack_rxmit)
goto send;
}
/*
* Compare available window to amount of window
* known to peer (as advertised window less
* next expected input). If the difference is at least two
* max size segments, or at least 50% of the maximum possible
* window, then want to send a window update to peer.
* Skip this if the connection is in T/TCP half-open state.
*/
if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN)) {
/*
* "adv" is the amount we can increase the window,
* taking into account that we are limited by
* TCP_MAXWIN << tp->rcv_scale.
*/
long adv = min(recwin, (long)TCP_MAXWIN << tp->rcv_scale) -
(tp->rcv_adv - tp->rcv_nxt);
if (adv >= (long) (2 * tp->t_maxseg))
goto send;
if (2 * adv >= (long) so->so_rcv.sb_hiwat)
goto send;
}
/*
* Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
* is also a catch-all for the retransmit timer timeout case.
*/
if (tp->t_flags & TF_ACKNOW)
goto send;
if ((flags & TH_RST) ||
((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0))
goto send;
if (SEQ_GT(tp->snd_up, tp->snd_una))
goto send;
/*
* If our state indicates that FIN should be sent
* and we have not yet done so, then we need to send.
*/
if (flags & TH_FIN &&
((tp->t_flags & TF_SENTFIN) == 0 || tp->snd_nxt == tp->snd_una))
goto send;
/*
* In SACK, it is possible for tcp_output to fail to send a segment
* after the retransmission timer has been turned off. Make sure
* that the retransmission timer is set.
*/
if (tp->sack_enable && SEQ_GT(tp->snd_max, tp->snd_una) &&
!callout_active(tp->tt_rexmt) &&
!callout_active(tp->tt_persist)) {
callout_reset(tp->tt_rexmt, tp->t_rxtcur,
tcp_timer_rexmt, tp);
goto just_return;
}
/*
* TCP window updates are not reliable, rather a polling protocol
* using ``persist'' packets is used to insure receipt of window
* updates. The three ``states'' for the output side are:
* idle not doing retransmits or persists
* persisting to move a small or zero window
* (re)transmitting and thereby not persisting
*
* callout_active(tp->tt_persist)
* is true when we are in persist state.
* tp->t_force
* is set when we are called to send a persist packet.
* callout_active(tp->tt_rexmt)
* is set when we are retransmitting
* The output side is idle when both timers are zero.
*
* If send window is too small, there is data to transmit, and no
* retransmit or persist is pending, then go to persist state.
* If nothing happens soon, send when timer expires:
* if window is nonzero, transmit what we can,
* otherwise force out a byte.
*/
if (so->so_snd.sb_cc && !callout_active(tp->tt_rexmt) &&
!callout_active(tp->tt_persist)) {
tp->t_rxtshift = 0;
tcp_setpersist(tp);
}
/*
* No reason to send a segment, just return.
*/
just_return:
SOCKBUF_UNLOCK(&so->so_snd);
return (0);
send:
SOCKBUF_LOCK_ASSERT(&so->so_snd);
/*
* Before ESTABLISHED, force sending of initial options
* unless TCP set not to do any options.
* NOTE: we assume that the IP/TCP header plus TCP options
* always fit in a single mbuf, leaving room for a maximum
* link header, i.e.
* max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
*/
optlen = 0;
#ifdef INET6
if (isipv6)
hdrlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
else
#endif
hdrlen = sizeof (struct tcpiphdr);
if (flags & TH_SYN) {
tp->snd_nxt = tp->iss;
if ((tp->t_flags & TF_NOOPT) == 0) {
u_short mss;
opt[0] = TCPOPT_MAXSEG;
opt[1] = TCPOLEN_MAXSEG;
mss = htons((u_short) tcp_mssopt(&tp->t_inpcb->inp_inc));
(void)memcpy(opt + 2, &mss, sizeof(mss));
optlen = TCPOLEN_MAXSEG;
/*
* If this is the first SYN of connection (not a SYN
* ACK), include SACK_PERMIT_HDR option. If this is a
* SYN ACK, include SACK_PERMIT_HDR option if peer has
* already done so. This is only for active connect,
* since the syncache takes care of the passive connect.
*/
if (tp->sack_enable && ((flags & TH_ACK) == 0 ||
(tp->t_flags & TF_SACK_PERMIT))) {
*((u_int32_t *) (opt + optlen)) =
htonl(TCPOPT_SACK_PERMIT_HDR);
optlen += 4;
}
if ((tp->t_flags & TF_REQ_SCALE) &&
((flags & TH_ACK) == 0 ||
(tp->t_flags & TF_RCVD_SCALE))) {
*((u_int32_t *)(opt + optlen)) = htonl(
TCPOPT_NOP << 24 |
TCPOPT_WINDOW << 16 |
TCPOLEN_WINDOW << 8 |
tp->request_r_scale);
optlen += 4;
}
}
}
/*
* Send a timestamp and echo-reply if this is a SYN and our side
* wants to use timestamps (TF_REQ_TSTMP is set) or both our side
* and our peer have sent timestamps in our SYN's.
*/
if ((tp->t_flags & (TF_REQ_TSTMP|TF_NOOPT)) == TF_REQ_TSTMP &&
(flags & TH_RST) == 0 &&
((flags & TH_ACK) == 0 ||
(tp->t_flags & TF_RCVD_TSTMP))) {
u_int32_t *lp = (u_int32_t *)(opt + optlen);
/* Form timestamp option as shown in appendix A of RFC 1323. */
*lp++ = htonl(TCPOPT_TSTAMP_HDR);
*lp++ = htonl(ticks);
*lp = htonl(tp->ts_recent);
optlen += TCPOLEN_TSTAMP_APPA;
}
/*
* Send SACKs if necessary. This should be the last option processed.
* Only as many SACKs are sent as are permitted by the maximum options
* size. No more than three SACKs are sent.
*/
if (tp->sack_enable && tp->t_state == TCPS_ESTABLISHED &&
(tp->t_flags & (TF_SACK_PERMIT|TF_NOOPT)) == TF_SACK_PERMIT &&
tp->rcv_numsacks) {
u_int32_t *lp = (u_int32_t *)(opt + optlen);
u_int32_t *olp = lp++;
int count = 0; /* actual number of SACKs inserted */
int maxsack = (MAX_TCPOPTLEN - (optlen + 4))/TCPOLEN_SACK;
tcpstat.tcps_sack_send_blocks++;
maxsack = min(maxsack, TCP_MAX_SACK);
for (i = 0; (i < tp->rcv_numsacks && count < maxsack); i++) {
struct sackblk sack = tp->sackblks[i];
if (sack.start == 0 && sack.end == 0)
continue;
*lp++ = htonl(sack.start);
*lp++ = htonl(sack.end);
count++;
}
*olp = htonl(TCPOPT_SACK_HDR|(TCPOLEN_SACK*count+2));
optlen += TCPOLEN_SACK*count + 4; /* including leading NOPs */
}
/*
* Send `CC-family' options if our side wants to use them (TF_REQ_CC),
* options are allowed (!TF_NOOPT) and it's not a RST.
*/
if ((tp->t_flags & (TF_REQ_CC|TF_NOOPT)) == TF_REQ_CC &&
(flags & TH_RST) == 0) {
switch (flags & (TH_SYN|TH_ACK)) {
/*
* This is a normal ACK, send CC if we received CC before
* from our peer.
*/
case TH_ACK:
if (!(tp->t_flags & TF_RCVD_CC))
break;
/*FALLTHROUGH*/
/*
* We can only get here in T/TCP's SYN_SENT* state, when
* we're a sending a non-SYN segment without waiting for
* the ACK of our SYN. A check above assures that we only
* do this if our peer understands T/TCP.
*/
case 0:
opt[optlen++] = TCPOPT_NOP;
opt[optlen++] = TCPOPT_NOP;
opt[optlen++] = TCPOPT_CC;
opt[optlen++] = TCPOLEN_CC;
*(u_int32_t *)&opt[optlen] = htonl(tp->cc_send);
optlen += 4;
break;
/*
* This is our initial SYN, check whether we have to use
* CC or CC.new.
*/
case TH_SYN:
opt[optlen++] = TCPOPT_NOP;
opt[optlen++] = TCPOPT_NOP;
opt[optlen++] = tp->t_flags & TF_SENDCCNEW ?
TCPOPT_CCNEW : TCPOPT_CC;
opt[optlen++] = TCPOLEN_CC;
*(u_int32_t *)&opt[optlen] = htonl(tp->cc_send);
optlen += 4;
break;
/*
* This is a SYN,ACK; send CC and CC.echo if we received
* CC from our peer.
*/
case (TH_SYN|TH_ACK):
if (tp->t_flags & TF_RCVD_CC) {
opt[optlen++] = TCPOPT_NOP;
opt[optlen++] = TCPOPT_NOP;
opt[optlen++] = TCPOPT_CC;
opt[optlen++] = TCPOLEN_CC;
*(u_int32_t *)&opt[optlen] =
htonl(tp->cc_send);
optlen += 4;
opt[optlen++] = TCPOPT_NOP;
opt[optlen++] = TCPOPT_NOP;
opt[optlen++] = TCPOPT_CCECHO;
opt[optlen++] = TCPOLEN_CC;
*(u_int32_t *)&opt[optlen] =
htonl(tp->cc_recv);
optlen += 4;
}
break;
}
}
#ifdef TCP_SIGNATURE
#ifdef INET6
if (!isipv6)
#endif
if (tp->t_flags & TF_SIGNATURE) {
int i;
u_char *bp;
/* Initialize TCP-MD5 option (RFC2385) */
bp = (u_char *)opt + optlen;
*bp++ = TCPOPT_SIGNATURE;
*bp++ = TCPOLEN_SIGNATURE;
sigoff = optlen + 2;
for (i = 0; i < TCP_SIGLEN; i++)
*bp++ = 0;
optlen += TCPOLEN_SIGNATURE;
/* Terminate options list and maintain 32-bit alignment. */
*bp++ = TCPOPT_NOP;
*bp++ = TCPOPT_EOL;
optlen += 2;
}
#endif /* TCP_SIGNATURE */
hdrlen += optlen;
#ifdef INET6
if (isipv6)
ipoptlen = ip6_optlen(tp->t_inpcb);
else
#endif
if (tp->t_inpcb->inp_options)
ipoptlen = tp->t_inpcb->inp_options->m_len -
offsetof(struct ipoption, ipopt_list);
else
