static int
do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
uint32_t vld;
struct toepcb *toep = lookup_tid(sc, tid);
struct tom_data *td = toep->td;
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
KASSERT(!(toep->flags & TPF_SYNQE),
("%s: toep %p claims to be a synq entry", __func__, toep));
vld = be32toh(cpl->ddpvld);
if (__predict_false(vld & DDP_ERR)) {
panic("%s: DDP error 0x%x (tid %d, toep %p)",
__func__, vld, tid, toep);
}
if (toep->ulp_mode == ULP_MODE_ISCSI) {
m = m_get(M_NOWAIT, MT_DATA);
if (m == NULL)
CXGBE_UNIMPLEMENTED("mbuf alloc failure");
memcpy(mtod(m, unsigned char *), cpl,
sizeof(struct cpl_rx_data_ddp));
if (!t4_cpl_iscsi_callback(td, toep, m, CPL_RX_DATA_DDP))
return (0);
m_freem(m);
}
static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_pdu *ip;
struct icl_cxgbei_pdu *icp;
M_ASSERTPKTHDR(m);
ip = icl_cxgbei_new_pdu(M_NOWAIT);
if (ip == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
icp = ip_to_icp(ip);
bcopy(mtod(m, caddr_t) + sizeof(*cpl), icp->ip.ip_bhs, sizeof(struct
iscsi_bhs));
icp->pdu_seq = ntohl(cpl->seq);
icp->pdu_flags = SBUF_ULP_FLAG_HDR_RCVD;
/* This is the start of a new PDU. There should be no old state. */
MPASS(toep->ulpcb2 == NULL);
toep->ulpcb2 = icp;
#if 0
CTR4(KTR_CXGBE, "%s: tid %u, cpl->len hlen %u, m->m_len hlen %u",
__func__, tid, ntohs(cpl->len), m->m_len);
#endif
m_freem(m);
return (0);
}
static int
do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_SET_TCB_RPL,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
if (tid >= sc->tids.ftid_base &&
tid < sc->tids.ftid_base + sc->tids.nftids)
return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
CXGBE_UNIMPLEMENTED(__func__);
}
static int
ppod_write_idata(struct cxgbei_data *ci,
struct cxgbei_ulp2_pagepod_hdr *hdr,
unsigned int idx, unsigned int npods,
struct cxgbei_ulp2_gather_list *gl,
unsigned int gl_pidx, struct toepcb *toep)
{
u_int dlen = PPOD_SIZE * npods;
u_int pm_addr = idx * PPOD_SIZE + ci->llimit;
u_int wr_len = roundup(sizeof(struct ulp_mem_io) +
sizeof(struct ulptx_idata) + dlen, 16);
struct ulp_mem_io *req;
struct ulptx_idata *idata;
struct pagepod *ppod;
u_int i;
struct wrqe *wr;
struct adapter *sc = toep->vi->pi->adapter;
wr = alloc_wrqe(wr_len, toep->ctrlq);
if (wr == NULL) {
CXGBE_UNIMPLEMENTED("ppod_write_idata: alloc_wrqe failure");
return (ENOMEM);
}
req = wrtod(wr);
memset(req, 0, wr_len);
ulp_mem_io_set_hdr(sc, toep->tid, req, wr_len, dlen, pm_addr);
idata = (struct ulptx_idata *)(req + 1);
ppod = (struct pagepod *)(idata + 1);
for (i = 0; i < npods; i++, ppod++, gl_pidx += PPOD_PAGES) {
if (!hdr) /* clear the pagepod */
ppod_clear(ppod);
else /* set the pagepod */
ppod_set(ppod, hdr, gl, gl_pidx);
}
t4_wrq_tx(sc, wr);
return 0;
}
static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_pdu *ip;
struct icl_cxgbei_pdu *icp;
uint16_t len_ddp = be16toh(cpl->pdu_len_ddp);
uint16_t len = be16toh(cpl->len);
M_ASSERTPKTHDR(m);
MPASS(m->m_pkthdr.len == len + sizeof(*cpl));
ip = icl_cxgbei_new_pdu(M_NOWAIT);
if (ip == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs);
ip->ip_data_len = G_ISCSI_PDU_LEN(len_ddp) - len;
icp = ip_to_icp(ip);
icp->icp_seq = ntohl(cpl->seq);
icp->icp_flags = ICPF_RX_HDR;
/* This is the start of a new PDU. There should be no old state. */
MPASS(toep->ulpcb2 == NULL);
toep->ulpcb2 = icp;
#if 0
CTR5(KTR_CXGBE, "%s: tid %u, cpl->len %u, pdu_len_ddp 0x%04x, icp %p",
__func__, tid, len, len_ddp, icp);
#endif
m_freem(m);
return (0);
}
static int
do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cxgbei_data *ci = sc->iscsi_ulp_softc;
const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct socket *so;
struct sockbuf *sb;
struct tcpcb *tp;
struct icl_cxgbei_conn *icc;
struct icl_conn *ic;
struct icl_cxgbei_pdu *icp = toep->ulpcb2;
struct icl_pdu *ip;
u_int pdu_len, val;
MPASS(m == NULL);
/* Must already be assembling a PDU. */
MPASS(icp != NULL);
MPASS(icp->icp_flags & ICPF_RX_HDR); /* Data is optional. */
MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0);
pdu_len = be16toh(cpl->len); /* includes everything. */
val = be32toh(cpl->ddpvld);
#if 0
CTR5(KTR_CXGBE,
"%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp_flags 0x%08x",
__func__, tid, pdu_len, val, icp->icp_flags);
#endif
icp->icp_flags |= ICPF_RX_STATUS;
ip = &icp->ip;
if (val & F_DDP_PADDING_ERR)
icp->icp_flags |= ICPF_PAD_ERR;
if (val & F_DDP_HDRCRC_ERR)
icp->icp_flags |= ICPF_HCRC_ERR;
if (val & F_DDP_DATACRC_ERR)
icp->icp_flags |= ICPF_DCRC_ERR;
if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) {
MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0);
MPASS(ip->ip_data_len > 0);
icp->icp_flags |= ICPF_RX_DDP;
counter_u64_add(ci->ddp_pdus, 1);
counter_u64_add(ci->ddp_bytes, ip->ip_data_len);
}
INP_WLOCK(inp);
if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
__func__, tid, pdu_len, inp->inp_flags);
INP_WUNLOCK(inp);
icl_cxgbei_conn_pdu_free(NULL, ip);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
tp = intotcpcb(inp);
MPASS(icp->icp_seq == tp->rcv_nxt);
MPASS(tp->rcv_wnd >= pdu_len);
tp->rcv_nxt += pdu_len;
tp->rcv_wnd -= pdu_len;
tp->t_rcvtime = ticks;
/* update rx credits */
toep->rx_credits += pdu_len;
t4_rcvd(&toep->td->tod, tp); /* XXX: sc->tom_softc.tod */
so = inp->inp_socket;
sb = &so->so_rcv;
SOCKBUF_LOCK(sb);
icc = toep->ulpcb;
if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) {
CTR5(KTR_CXGBE,
"%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
__func__, tid, pdu_len, icc, sb->sb_state);
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
INP_INFO_RLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = tcp_drop(tp, ECONNRESET);
if (tp)
INP_WUNLOCK(inp);
INP_INFO_RUNLOCK(&V_tcbinfo);
icl_cxgbei_conn_pdu_free(NULL, ip);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
ic = &icc->ic;
icl_cxgbei_new_pdu_set_conn(ip, ic);
MPASS(m == NULL); /* was unused, we'll use it now. */
m = sbcut_locked(sb, sbused(sb)); /* XXXNP: toep->sb_cc accounting? */
if (__predict_false(m != NULL)) {
int len = m_length(m, NULL);
/*
* PDUs were received before the tid transitioned to ULP mode.
* Convert them to icl_cxgbei_pdus and send them to ICL before
* the PDU in icp/ip.
*/
CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, tid,
len);
/* XXXNP: needs to be rewritten. */
if (len == sizeof(struct iscsi_bhs) || len == 4 + sizeof(struct
iscsi_bhs)) {
struct icl_cxgbei_pdu *icp0;
struct icl_pdu *ip0;
ip0 = icl_cxgbei_new_pdu(M_NOWAIT);
icl_cxgbei_new_pdu_set_conn(ip0, ic);
if (ip0 == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
icp0 = ip_to_icp(ip0);
icp0->icp_seq = 0; /* XXX */
icp0->icp_flags = ICPF_RX_HDR | ICPF_RX_STATUS;
m_copydata(m, 0, sizeof(struct iscsi_bhs), (void *)ip0->ip_bhs);
STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip0, ip_next);
}
m_freem(m);
}
STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
if ((icc->rx_flags & RXF_ACTIVE) == 0) {
struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt];
mtx_lock(&cwt->cwt_lock);
icc->rx_flags |= RXF_ACTIVE;
TAILQ_INSERT_TAIL(&cwt->rx_head, icc, rx_link);
if (cwt->cwt_state == CWT_SLEEPING) {
cwt->cwt_state = CWT_RUNNING;
cv_signal(&cwt->cwt_cv);
}
mtx_unlock(&cwt->cwt_lock);
}
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
static int
handle_ddp_data(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt, int len)
{
uint32_t report = be32toh(ddp_report);
unsigned int db_flag;
struct inpcb *inp = toep->inp;
struct tcpcb *tp;
struct socket *so;
struct sockbuf *sb;
struct mbuf *m;
db_flag = report & F_DDP_BUF_IDX ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE;
if (__predict_false(!(report & F_DDP_INV)))
CXGBE_UNIMPLEMENTED("DDP buffer still valid");
INP_WLOCK(inp);
so = inp_inpcbtosocket(inp);
sb = &so->so_rcv;
if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
/*
* XXX: think a bit more.
* tcpcb probably gone, but socket should still be around
* because we always wait for DDP completion in soreceive no
* matter what. Just wake it up and let it clean up.
*/
CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
__func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
SOCKBUF_LOCK(sb);
goto wakeup;
}
tp = intotcpcb(inp);
len += be32toh(rcv_nxt) - tp->rcv_nxt;
tp->rcv_nxt += len;
tp->t_rcvtime = ticks;
#ifndef USE_DDP_RX_FLOW_CONTROL
KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
tp->rcv_wnd -= len;
#endif
m = get_ddp_mbuf(len);
SOCKBUF_LOCK(sb);
if (report & F_DDP_BUF_COMPLETE)
toep->ddp_score = DDP_HIGH_SCORE;
else
discourage_ddp(toep);
/* receive buffer autosize */
if (sb->sb_flags & SB_AUTOSIZE &&
V_tcp_do_autorcvbuf &&
sb->sb_hiwat < V_tcp_autorcvbuf_max &&
len > (sbspace(sb) / 8 * 7)) {
unsigned int hiwat = sb->sb_hiwat;
unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc,
V_tcp_autorcvbuf_max);
if (!sbreserve_locked(sb, newsize, so, NULL))
sb->sb_flags &= ~SB_AUTOSIZE;
else
toep->rx_credits += newsize - hiwat;
}
KASSERT(toep->sb_cc >= sbused(sb),
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sbused(sb), toep->sb_cc));
toep->rx_credits += toep->sb_cc - sbused(sb);
#ifdef USE_DDP_RX_FLOW_CONTROL
toep->rx_credits -= len; /* adjust for F_RX_FC_DDP */
#endif
sbappendstream_locked(sb, m, 0);
toep->sb_cc = sbused(sb);
wakeup:
KASSERT(toep->ddp_flags & db_flag,
("%s: DDP buffer not active. toep %p, ddp_flags 0x%x, report 0x%x",
__func__, toep, toep->ddp_flags, report));
toep->ddp_flags &= ~db_flag;
sorwakeup_locked(so);
SOCKBUF_UNLOCK_ASSERT(sb);
INP_WUNLOCK(inp);
return (0);
}
static int
alloc_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
{
int rc, cntxt_id;
__be32 v;
struct adapter *sc = pi->adapter;
struct netmap_adapter *na = NA(pi->nm_ifp);
struct fw_iq_cmd c;
MPASS(na != NULL);
MPASS(nm_rxq->iq_desc != NULL);
MPASS(nm_rxq->fl_desc != NULL);
bzero(nm_rxq->iq_desc, pi->qsize_rxq * IQ_ESIZE);
bzero(nm_rxq->fl_desc, na->num_rx_desc * EQ_ESIZE + spg_len);
bzero(&c, sizeof(c));
c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
V_FW_IQ_CMD_VFN(0));
c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
FW_LEN16(c));
if (pi->flags & INTR_NM_RXQ) {
KASSERT(nm_rxq->intr_idx < sc->intr_count,
("%s: invalid direct intr_idx %d", __func__,
nm_rxq->intr_idx));
v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx);
} else {
CXGBE_UNIMPLEMENTED(__func__); /* XXXNM: needs review */
v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx) |
F_FW_IQ_CMD_IQANDST;
}
c.type_to_iqandstindex = htobe32(v |
V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
V_FW_IQ_CMD_VIID(pi->nm_viid) |
V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
F_FW_IQ_CMD_IQGTSMODE |
V_FW_IQ_CMD_IQINTCNTTHRESH(0) |
V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4));
c.iqsize = htobe16(pi->qsize_rxq);
c.iqaddr = htobe64(nm_rxq->iq_ba);
c.iqns_to_fl0congen |=
htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
(fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0));
c.fl0dcaen_to_fl0cidxfthresh =
htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
c.fl0size = htobe16(na->num_rx_desc + spg_len / EQ_ESIZE);
c.fl0addr = htobe64(nm_rxq->fl_ba);
rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
if (rc != 0) {
device_printf(sc->dev,
"failed to create netmap ingress queue: %d\n", rc);
return (rc);
}
nm_rxq->iq_cidx = 0;
MPASS(nm_rxq->iq_sidx == pi->qsize_rxq - spg_len / IQ_ESIZE);
nm_rxq->iq_gen = F_RSPD_GEN;
nm_rxq->iq_cntxt_id = be16toh(c.iqid);
nm_rxq->iq_abs_id = be16toh(c.physiqid);
cntxt_id = nm_rxq->iq_cntxt_id - sc->sge.iq_start;
if (cntxt_id >= sc->sge.niq) {
panic ("%s: nm_rxq->iq_cntxt_id (%d) more than the max (%d)",
__func__, cntxt_id, sc->sge.niq - 1);
}
sc->sge.iqmap[cntxt_id] = (void *)nm_rxq;
nm_rxq->fl_cntxt_id = be16toh(c.fl0id);
nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;
MPASS(nm_rxq->fl_sidx == na->num_rx_desc);
cntxt_id = nm_rxq->fl_cntxt_id - sc->sge.eq_start;
if (cntxt_id >= sc->sge.neq) {
panic("%s: nm_rxq->fl_cntxt_id (%d) more than the max (%d)",
__func__, cntxt_id, sc->sge.neq - 1);
}
sc->sge.eqmap[cntxt_id] = (void *)nm_rxq;
nm_rxq->fl_db_val = F_DBPRIO | V_QID(nm_rxq->fl_cntxt_id) | V_PIDX(0);
if (is_t5(sc))
nm_rxq->fl_db_val |= F_DBTYPE;
t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(F_QINTR_CNT_EN) |
V_INGRESSQID(nm_rxq->iq_cntxt_id));
return (rc);
}
/*
* Peer has sent us a FIN.
*/
static int
do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_peer_close *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct tcpcb *tp = NULL;
struct socket *so;
struct sockbuf *sb;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_PEER_CLOSE,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
if (__predict_false(toep->flags & TPF_SYNQE)) {
#ifdef INVARIANTS
struct synq_entry *synqe = (void *)toep;
INP_WLOCK(synqe->lctx->inp);
if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
("%s: listen socket closed but tid %u not aborted.",
__func__, tid));
} else {
/*
* do_pass_accept_req is still running and will
* eventually take care of this tid.
*/
}
INP_WUNLOCK(synqe->lctx->inp);
#endif
CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
toep, toep->flags);
return (0);
}
KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
INP_INFO_WLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = intotcpcb(inp);
CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__,
tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp);
if (toep->flags & TPF_ABORT_SHUTDOWN)
goto done;
tp->rcv_nxt++; /* FIN */
so = inp->inp_socket;
sb = &so->so_rcv;
SOCKBUF_LOCK(sb);
if (__predict_false(toep->ddp_flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) {
m = m_get(M_NOWAIT, MT_DATA);
if (m == NULL)
CXGBE_UNIMPLEMENTED("mbuf alloc failure");
m->m_len = be32toh(cpl->rcv_nxt) - tp->rcv_nxt;
m->m_flags |= M_DDP; /* Data is already where it should be */
m->m_data = "nothing to see here";
tp->rcv_nxt = be32toh(cpl->rcv_nxt);
toep->ddp_flags &= ~(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE);
KASSERT(toep->sb_cc >= sb->sb_cc,
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sb->sb_cc, toep->sb_cc));
toep->rx_credits += toep->sb_cc - sb->sb_cc;
#ifdef USE_DDP_RX_FLOW_CONTROL
toep->rx_credits -= m->m_len; /* adjust for F_RX_FC_DDP */
#endif
sbappendstream_locked(sb, m);
toep->sb_cc = sb->sb_cc;
}
socantrcvmore_locked(so); /* unlocks the sockbuf */
KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),
("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
be32toh(cpl->rcv_nxt)));
switch (tp->t_state) {
case TCPS_SYN_RECEIVED:
tp->t_starttime = ticks;
/* FALLTHROUGH */
case TCPS_ESTABLISHED:
tp->t_state = TCPS_CLOSE_WAIT;
break;
case TCPS_FIN_WAIT_1:
tp->t_state = TCPS_CLOSING;
break;
case TCPS_FIN_WAIT_2:
tcp_twstart(tp);
INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
INP_INFO_WUNLOCK(&V_tcbinfo);
INP_WLOCK(inp);
final_cpl_received(toep);
return (0);
default:
log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
__func__, tid, tp->t_state);
}
done:
INP_WUNLOCK(inp);
INP_INFO_WUNLOCK(&V_tcbinfo);
return (0);
}
/*
* Send data and/or a FIN to the peer.
*
* The socket's so_snd buffer consists of a stream of data starting with sb_mb
* and linked together with m_next. sb_sndptr, if set, is the last mbuf that
* was transmitted.
*/
static void
t4_push_frames(struct adapter *sc, struct toepcb *toep)
{
struct mbuf *sndptr, *m, *sb_sndptr;
struct fw_ofld_tx_data_wr *txwr;
struct wrqe *wr;
unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
struct inpcb *inp = toep->inp;
struct tcpcb *tp = intotcpcb(inp);
struct socket *so = inp->inp_socket;
struct sockbuf *sb = &so->so_snd;
int tx_credits;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
INP_WLOCK_ASSERT(inp);
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
if (__predict_false(toep->ulp_mode != ULP_MODE_NONE &&
toep->ulp_mode != ULP_MODE_TCPDDP))
CXGBE_UNIMPLEMENTED("ulp_mode");
/*
* This function doesn't resume by itself. Someone else must clear the
* flag and call this function.
*/
if (__predict_false(toep->flags & TPF_TX_SUSPENDED))
return;
do {
tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
max_imm = max_imm_payload(tx_credits);
max_nsegs = max_dsgl_nsegs(tx_credits);
SOCKBUF_LOCK(sb);
sb_sndptr = sb->sb_sndptr;
sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb;
plen = 0;
nsegs = 0;
max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
for (m = sndptr; m != NULL; m = m->m_next) {
int n = sglist_count(mtod(m, void *), m->m_len);
nsegs += n;
plen += m->m_len;
/* This mbuf sent us _over_ the nsegs limit, back out */
if (plen > max_imm && nsegs > max_nsegs) {
nsegs -= n;
plen -= m->m_len;
if (plen == 0) {
/* Too few credits */
toep->flags |= TPF_TX_SUSPENDED;
SOCKBUF_UNLOCK(sb);
return;
}
break;
}
if (max_nsegs_1mbuf < n)
max_nsegs_1mbuf = n;
sb_sndptr = m; /* new sb->sb_sndptr if all goes well */
/* This mbuf put us right at the max_nsegs limit */
if (plen > max_imm && nsegs == max_nsegs) {
m = m->m_next;
break;
}
}
if (sb->sb_flags & SB_AUTOSIZE &&
V_tcp_do_autosndbuf &&
sb->sb_hiwat < V_tcp_autosndbuf_max &&
sbspace(sb) < sb->sb_hiwat / 8 * 7) {
int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
V_tcp_autosndbuf_max);
if (!sbreserve_locked(sb, newsize, so, NULL))
sb->sb_flags &= ~SB_AUTOSIZE;
else {
sowwakeup_locked(so); /* room available */
SOCKBUF_UNLOCK_ASSERT(sb);
goto unlocked;
}
}
SOCKBUF_UNLOCK(sb);
unlocked:
/* nothing to send */
if (plen == 0) {
KASSERT(m == NULL,
("%s: nothing to send, but m != NULL", __func__));
break;
}
if (__predict_false(toep->flags & TPF_FIN_SENT))
panic("%s: excess tx.", __func__);
if (plen <= max_imm) {
/* Immediate data tx */
wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16),
toep->ofld_txq);
if (wr == NULL) {
/* XXX: how will we recover from this? */
toep->flags |= TPF_TX_SUSPENDED;
return;
}
txwr = wrtod(wr);
credits = howmany(wr->wr_len, 16);
write_tx_wr(txwr, toep, plen, plen, credits,
tp->t_flags & TF_MORETOCOME);
m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
} else {
int wr_len;
/* DSGL tx */
wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq);
if (wr == NULL) {
/* XXX: how will we recover from this? */
toep->flags |= TPF_TX_SUSPENDED;
return;
}
txwr = wrtod(wr);
credits = howmany(wr_len, 16);
write_tx_wr(txwr, toep, 0, plen, credits,
tp->t_flags & TF_MORETOCOME);
write_tx_sgl(txwr + 1, sndptr, m, nsegs,
max_nsegs_1mbuf);
if (wr_len & 0xf) {
uint64_t *pad = (uint64_t *)
((uintptr_t)txwr + wr_len);
*pad = 0;
}
}
KASSERT(toep->tx_credits >= credits,
("%s: not enough credits", __func__));
toep->tx_credits -= credits;
tp->snd_nxt += plen;
tp->snd_max += plen;
SOCKBUF_LOCK(sb);
KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__));
sb->sb_sndptr = sb_sndptr;
SOCKBUF_UNLOCK(sb);
toep->flags |= TPF_TX_DATA_SENT;
KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
txsd->plen = plen;
txsd->tx_credits = credits;
txsd++;
if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
toep->txsd_pidx = 0;
txsd = &toep->txsd[0];
}
toep->txsd_avail--;
t4_l2t_send(sc, wr, toep->l2te);
} while (m != NULL);
/* Send a FIN if requested, but only if there's no more data to send */
if (m == NULL && toep->flags & TPF_SEND_FIN)
close_conn(sc, toep);
}
