void
handle_ddp_close(struct toepcb *toep, struct tcpcb *tp, struct sockbuf *sb,
__be32 rcv_nxt)
{
struct mbuf *m;
int len;
SOCKBUF_LOCK_ASSERT(sb);
INP_WLOCK_ASSERT(toep->inp);
len = be32toh(rcv_nxt) - tp->rcv_nxt;
/* Signal handle_ddp() to break out of its sleep loop. */
toep->ddp_flags &= ~(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE);
if (len == 0)
return;
tp->rcv_nxt += len;
KASSERT(toep->sb_cc >= sbused(sb),
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sbused(sb), toep->sb_cc));
toep->rx_credits += toep->sb_cc - sbused(sb);
#ifdef USE_DDP_RX_FLOW_CONTROL
toep->rx_credits -= len; /* adjust for F_RX_FC_DDP */
#endif
m = get_ddp_mbuf(len);
sbappendstream_locked(sb, m, 0);
toep->sb_cc = sbused(sb);
}
/* XXX: handle_ddp_data code duplication */
void
insert_ddp_data(struct toepcb *toep, uint32_t n)
{
struct inpcb *inp = toep->inp;
struct tcpcb *tp = intotcpcb(inp);
struct sockbuf *sb = &inp->inp_socket->so_rcv;
struct mbuf *m;
INP_WLOCK_ASSERT(inp);
SOCKBUF_LOCK_ASSERT(sb);
m = get_ddp_mbuf(n);
tp->rcv_nxt += n;
#ifndef USE_DDP_RX_FLOW_CONTROL
KASSERT(tp->rcv_wnd >= n, ("%s: negative window size", __func__));
tp->rcv_wnd -= n;
#endif
KASSERT(toep->sb_cc >= sbused(sb),
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sbused(sb), toep->sb_cc));
toep->rx_credits += toep->sb_cc - sbused(sb);
#ifdef USE_DDP_RX_FLOW_CONTROL
toep->rx_credits -= n; /* adjust for F_RX_FC_DDP */
#endif
sbappendstream_locked(sb, m, 0);
toep->sb_cc = sbused(sb);
}
static int
handle_ddp_data(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt, int len)
{
uint32_t report = be32toh(ddp_report);
unsigned int db_flag;
struct inpcb *inp = toep->inp;
struct tcpcb *tp;
struct socket *so;
struct sockbuf *sb;
struct mbuf *m;
db_flag = report & F_DDP_BUF_IDX ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE;
if (__predict_false(!(report & F_DDP_INV)))
CXGBE_UNIMPLEMENTED("DDP buffer still valid");
INP_WLOCK(inp);
so = inp_inpcbtosocket(inp);
sb = &so->so_rcv;
if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
/*
* XXX: think a bit more.
* tcpcb probably gone, but socket should still be around
* because we always wait for DDP completion in soreceive no
* matter what. Just wake it up and let it clean up.
*/
CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
__func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
SOCKBUF_LOCK(sb);
goto wakeup;
}
tp = intotcpcb(inp);
len += be32toh(rcv_nxt) - tp->rcv_nxt;
tp->rcv_nxt += len;
tp->t_rcvtime = ticks;
#ifndef USE_DDP_RX_FLOW_CONTROL
KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
tp->rcv_wnd -= len;
#endif
m = get_ddp_mbuf(len);
SOCKBUF_LOCK(sb);
if (report & F_DDP_BUF_COMPLETE)
toep->ddp_score = DDP_HIGH_SCORE;
else
discourage_ddp(toep);
/* receive buffer autosize */
if (sb->sb_flags & SB_AUTOSIZE &&
V_tcp_do_autorcvbuf &&
sb->sb_hiwat < V_tcp_autorcvbuf_max &&
len > (sbspace(sb) / 8 * 7)) {
unsigned int hiwat = sb->sb_hiwat;
unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc,
V_tcp_autorcvbuf_max);
if (!sbreserve_locked(sb, newsize, so, NULL))
sb->sb_flags &= ~SB_AUTOSIZE;
else
toep->rx_credits += newsize - hiwat;
}
KASSERT(toep->sb_cc >= sbused(sb),
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sbused(sb), toep->sb_cc));
toep->rx_credits += toep->sb_cc - sbused(sb);
#ifdef USE_DDP_RX_FLOW_CONTROL
toep->rx_credits -= len; /* adjust for F_RX_FC_DDP */
#endif
sbappendstream_locked(sb, m, 0);
toep->sb_cc = sbused(sb);
wakeup:
KASSERT(toep->ddp_flags & db_flag,
("%s: DDP buffer not active. toep %p, ddp_flags 0x%x, report 0x%x",
__func__, toep, toep->ddp_flags, report));
toep->ddp_flags &= ~db_flag;
sorwakeup_locked(so);
SOCKBUF_UNLOCK_ASSERT(sb);
INP_WUNLOCK(inp);
return (0);
}
/*
* Peer has sent us a FIN.
*/
static int
do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_peer_close *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct tcpcb *tp = NULL;
struct socket *so;
struct sockbuf *sb;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_PEER_CLOSE,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
if (__predict_false(toep->flags & TPF_SYNQE)) {
#ifdef INVARIANTS
struct synq_entry *synqe = (void *)toep;
INP_WLOCK(synqe->lctx->inp);
if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
("%s: listen socket closed but tid %u not aborted.",
__func__, tid));
} else {
/*
* do_pass_accept_req is still running and will
* eventually take care of this tid.
*/
}
INP_WUNLOCK(synqe->lctx->inp);
#endif
CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
toep, toep->flags);
return (0);
}
KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
INP_INFO_WLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = intotcpcb(inp);
CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__,
tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp);
if (toep->flags & TPF_ABORT_SHUTDOWN)
goto done;
tp->rcv_nxt++; /* FIN */
so = inp->inp_socket;
sb = &so->so_rcv;
SOCKBUF_LOCK(sb);
if (__predict_false(toep->ddp_flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) {
m = get_ddp_mbuf(be32toh(cpl->rcv_nxt) - tp->rcv_nxt);
tp->rcv_nxt = be32toh(cpl->rcv_nxt);
toep->ddp_flags &= ~(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE);
KASSERT(toep->sb_cc >= sb->sb_cc,
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sb->sb_cc, toep->sb_cc));
toep->rx_credits += toep->sb_cc - sb->sb_cc;
#ifdef USE_DDP_RX_FLOW_CONTROL
toep->rx_credits -= m->m_len; /* adjust for F_RX_FC_DDP */
#endif
sbappendstream_locked(sb, m);
toep->sb_cc = sb->sb_cc;
}
socantrcvmore_locked(so); /* unlocks the sockbuf */
if (toep->ulp_mode != ULP_MODE_RDMA) {
KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),
("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
be32toh(cpl->rcv_nxt)));
}
switch (tp->t_state) {
case TCPS_SYN_RECEIVED:
tp->t_starttime = ticks;
/* FALLTHROUGH */
case TCPS_ESTABLISHED:
tp->t_state = TCPS_CLOSE_WAIT;
break;
case TCPS_FIN_WAIT_1:
tp->t_state = TCPS_CLOSING;
break;
case TCPS_FIN_WAIT_2:
tcp_twstart(tp);
INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
INP_INFO_WUNLOCK(&V_tcbinfo);
INP_WLOCK(inp);
final_cpl_received(toep);
return (0);
default:
log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
__func__, tid, tp->t_state);
}
done:
INP_WUNLOCK(inp);
INP_INFO_WUNLOCK(&V_tcbinfo);
return (0);
}
