static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_pdu *ip;
struct icl_cxgbei_pdu *icp;
M_ASSERTPKTHDR(m);
ip = icl_cxgbei_new_pdu(M_NOWAIT);
if (ip == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
icp = ip_to_icp(ip);
bcopy(mtod(m, caddr_t) + sizeof(*cpl), icp->ip.ip_bhs, sizeof(struct
iscsi_bhs));
icp->pdu_seq = ntohl(cpl->seq);
icp->pdu_flags = SBUF_ULP_FLAG_HDR_RCVD;
/* This is the start of a new PDU. There should be no old state. */
MPASS(toep->ulpcb2 == NULL);
toep->ulpcb2 = icp;
#if 0
CTR4(KTR_CXGBE, "%s: tid %u, cpl->len hlen %u, m->m_len hlen %u",
__func__, tid, ntohs(cpl->len), m->m_len);
#endif
m_freem(m);
return (0);
}
/*
* Allocate icl_pdu with empty BHS to fill up by the caller.
*/
static struct icl_pdu *
icl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags)
{
struct icl_pdu *ip;
ip = icl_cxgbei_new_pdu(flags);
if (__predict_false(ip == NULL))
return (NULL);
icl_cxgbei_new_pdu_set_conn(ip, ic);
return (ip);
}
static int
do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_pdu *ip;
struct icl_cxgbei_pdu *icp;
uint16_t len_ddp = be16toh(cpl->pdu_len_ddp);
uint16_t len = be16toh(cpl->len);
M_ASSERTPKTHDR(m);
MPASS(m->m_pkthdr.len == len + sizeof(*cpl));
ip = icl_cxgbei_new_pdu(M_NOWAIT);
if (ip == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs);
ip->ip_data_len = G_ISCSI_PDU_LEN(len_ddp) - len;
icp = ip_to_icp(ip);
icp->icp_seq = ntohl(cpl->seq);
icp->icp_flags = ICPF_RX_HDR;
/* This is the start of a new PDU. There should be no old state. */
MPASS(toep->ulpcb2 == NULL);
toep->ulpcb2 = icp;
#if 0
CTR5(KTR_CXGBE, "%s: tid %u, cpl->len %u, pdu_len_ddp 0x%04x, icp %p",
__func__, tid, len, len_ddp, icp);
#endif
m_freem(m);
return (0);
}
static int
do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cxgbei_data *ci = sc->iscsi_ulp_softc;
const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct socket *so;
struct sockbuf *sb;
struct tcpcb *tp;
struct icl_cxgbei_conn *icc;
struct icl_conn *ic;
struct icl_cxgbei_pdu *icp = toep->ulpcb2;
struct icl_pdu *ip;
u_int pdu_len, val;
MPASS(m == NULL);
/* Must already be assembling a PDU. */
MPASS(icp != NULL);
MPASS(icp->icp_flags & ICPF_RX_HDR); /* Data is optional. */
MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0);
pdu_len = be16toh(cpl->len); /* includes everything. */
val = be32toh(cpl->ddpvld);
#if 0
CTR5(KTR_CXGBE,
"%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp_flags 0x%08x",
__func__, tid, pdu_len, val, icp->icp_flags);
#endif
icp->icp_flags |= ICPF_RX_STATUS;
ip = &icp->ip;
if (val & F_DDP_PADDING_ERR)
icp->icp_flags |= ICPF_PAD_ERR;
if (val & F_DDP_HDRCRC_ERR)
icp->icp_flags |= ICPF_HCRC_ERR;
if (val & F_DDP_DATACRC_ERR)
icp->icp_flags |= ICPF_DCRC_ERR;
if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) {
MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0);
MPASS(ip->ip_data_len > 0);
icp->icp_flags |= ICPF_RX_DDP;
counter_u64_add(ci->ddp_pdus, 1);
counter_u64_add(ci->ddp_bytes, ip->ip_data_len);
}
INP_WLOCK(inp);
if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
__func__, tid, pdu_len, inp->inp_flags);
INP_WUNLOCK(inp);
icl_cxgbei_conn_pdu_free(NULL, ip);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
tp = intotcpcb(inp);
MPASS(icp->icp_seq == tp->rcv_nxt);
MPASS(tp->rcv_wnd >= pdu_len);
tp->rcv_nxt += pdu_len;
tp->rcv_wnd -= pdu_len;
tp->t_rcvtime = ticks;
/* update rx credits */
toep->rx_credits += pdu_len;
t4_rcvd(&toep->td->tod, tp); /* XXX: sc->tom_softc.tod */
so = inp->inp_socket;
sb = &so->so_rcv;
SOCKBUF_LOCK(sb);
icc = toep->ulpcb;
if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) {
CTR5(KTR_CXGBE,
"%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
__func__, tid, pdu_len, icc, sb->sb_state);
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
INP_INFO_RLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = tcp_drop(tp, ECONNRESET);
if (tp)
INP_WUNLOCK(inp);
INP_INFO_RUNLOCK(&V_tcbinfo);
icl_cxgbei_conn_pdu_free(NULL, ip);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
ic = &icc->ic;
icl_cxgbei_new_pdu_set_conn(ip, ic);
MPASS(m == NULL); /* was unused, we'll use it now. */
m = sbcut_locked(sb, sbused(sb)); /* XXXNP: toep->sb_cc accounting? */
if (__predict_false(m != NULL)) {
int len = m_length(m, NULL);
/*
* PDUs were received before the tid transitioned to ULP mode.
* Convert them to icl_cxgbei_pdus and send them to ICL before
* the PDU in icp/ip.
*/
CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, tid,
len);
/* XXXNP: needs to be rewritten. */
if (len == sizeof(struct iscsi_bhs) || len == 4 + sizeof(struct
iscsi_bhs)) {
struct icl_cxgbei_pdu *icp0;
struct icl_pdu *ip0;
ip0 = icl_cxgbei_new_pdu(M_NOWAIT);
icl_cxgbei_new_pdu_set_conn(ip0, ic);
if (ip0 == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
icp0 = ip_to_icp(ip0);
icp0->icp_seq = 0; /* XXX */
icp0->icp_flags = ICPF_RX_HDR | ICPF_RX_STATUS;
m_copydata(m, 0, sizeof(struct iscsi_bhs), (void *)ip0->ip_bhs);
STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip0, ip_next);
}
m_freem(m);
}
STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
if ((icc->rx_flags & RXF_ACTIVE) == 0) {
struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt];
mtx_lock(&cwt->cwt_lock);
icc->rx_flags |= RXF_ACTIVE;
TAILQ_INSERT_TAIL(&cwt->rx_head, icc, rx_link);
if (cwt->cwt_state == CWT_SLEEPING) {
cwt->cwt_state = CWT_RUNNING;
cv_signal(&cwt->cwt_cv);
}
mtx_unlock(&cwt->cwt_lock);
}
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}