/*
* Reply to the CPL_ABORT_REQ (send_reset)
*/
static int
do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_ABORT_RPL_RSS,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
if (toep->flags & TPF_SYNQE)
return (do_abort_rpl_synqe(iq, rss, m));
KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d",
__func__, tid, toep, inp, cpl->status);
KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
("%s: wasn't expecting abort reply", __func__));
INP_WLOCK(inp);
final_cpl_received(toep);
return (0);
}
void
act_open_failure_cleanup(struct adapter *sc, u_int atid, u_int status)
{
struct toepcb *toep = lookup_atid(sc, atid);
struct inpcb *inp = toep->inp;
struct toedev *tod = &toep->td->tod;
free_atid(sc, atid);
toep->tid = -1;
if (status != EAGAIN)
INP_INFO_RLOCK(&V_tcbinfo);
INP_WLOCK(inp);
toe_connect_failed(tod, inp, status);
final_cpl_received(toep); /* unlocks inp */
if (status != EAGAIN)
INP_INFO_RUNLOCK(&V_tcbinfo);
}
void
act_open_failure_cleanup(struct adapter *sc, u_int atid, u_int status)
{
struct toepcb *toep = lookup_atid(sc, atid);
struct inpcb *inp = toep->inp;
struct toedev *tod = &toep->td->tod;
struct epoch_tracker et;
free_atid(sc, atid);
toep->tid = -1;
CURVNET_SET(toep->vnet);
if (status != EAGAIN)
INP_INFO_RLOCK_ET(&V_tcbinfo, et);
INP_WLOCK(inp);
toe_connect_failed(tod, inp, status);
final_cpl_received(toep); /* unlocks inp */
if (status != EAGAIN)
INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
CURVNET_RESTORE();
}
static int
do_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
unsigned int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
unsigned int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
struct toepcb *toep = lookup_atid(sc, atid);
struct inpcb *inp = toep->inp;
struct toedev *tod = &toep->td->tod;
int rc;
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
KASSERT(toep->tid == atid, ("%s: toep tid/atid mismatch", __func__));
CTR3(KTR_CXGBE, "%s: atid %u, status %u ", __func__, atid, status);
/* Ignore negative advice */
if (negative_advice(status))
return (0);
free_atid(sc, atid);
toep->tid = -1;
if (status && act_open_has_tid(status))
release_tid(sc, GET_TID(cpl), toep->ctrlq);
rc = act_open_rpl_status_to_errno(status);
if (rc != EAGAIN)
INP_INFO_WLOCK(&V_tcbinfo);
INP_WLOCK(inp);
toe_connect_failed(tod, inp, rc);
final_cpl_received(toep); /* unlocks inp */
if (rc != EAGAIN)
INP_INFO_WUNLOCK(&V_tcbinfo);
return (0);
}
/*
* TCP RST from the peer, timeout, or some other such critical error.
*/
static int
do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct sge_wrq *ofld_txq = toep->ofld_txq;
struct inpcb *inp;
struct tcpcb *tp;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_ABORT_REQ_RSS,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
if (toep->flags & TPF_SYNQE)
return (do_abort_req_synqe(iq, rss, m));
KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
if (negative_advice(cpl->status)) {
CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)",
__func__, cpl->status, tid, toep->flags);
return (0); /* Ignore negative advice */
}
inp = toep->inp;
INP_INFO_WLOCK(&V_tcbinfo); /* for tcp_close */
INP_WLOCK(inp);
tp = intotcpcb(inp);
CTR6(KTR_CXGBE,
"%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d",
__func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
inp->inp_flags, cpl->status);
/*
* If we'd initiated an abort earlier the reply to it is responsible for
* cleaning up resources. Otherwise we tear everything down right here
* right now. We owe the T4 a CPL_ABORT_RPL no matter what.
*/
if (toep->flags & TPF_ABORT_SHUTDOWN) {
INP_WUNLOCK(inp);
goto done;
}
toep->flags |= TPF_ABORT_SHUTDOWN;
if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
struct socket *so = inp->inp_socket;
if (so != NULL)
so_error_set(so, abort_status_to_errno(tp,
cpl->status));
tp = tcp_close(tp);
if (tp == NULL)
INP_WLOCK(inp); /* re-acquire */
}
final_cpl_received(toep);
done:
INP_INFO_WUNLOCK(&V_tcbinfo);
send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST);
return (0);
}
/*
* Peer has ACK'd our FIN.
*/
static int
do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss,
struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct tcpcb *tp = NULL;
struct socket *so = NULL;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_CLOSE_CON_RPL,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
INP_INFO_WLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = intotcpcb(inp);
CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x",
__func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags);
if (toep->flags & TPF_ABORT_SHUTDOWN)
goto done;
so = inp->inp_socket;
tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */
switch (tp->t_state) {
case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */
tcp_twstart(tp);
release:
INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
INP_INFO_WUNLOCK(&V_tcbinfo);
INP_WLOCK(inp);
final_cpl_received(toep); /* no more CPLs expected */
return (0);
case TCPS_LAST_ACK:
if (tcp_close(tp))
INP_WUNLOCK(inp);
goto release;
case TCPS_FIN_WAIT_1:
if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
soisdisconnected(so);
tp->t_state = TCPS_FIN_WAIT_2;
break;
default:
log(LOG_ERR,
"%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n",
__func__, tid, tcpstates[tp->t_state]);
}
done:
INP_WUNLOCK(inp);
INP_INFO_WUNLOCK(&V_tcbinfo);
return (0);
}
/*
* Peer has sent us a FIN.
*/
static int
do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_peer_close *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct tcpcb *tp = NULL;
struct socket *so;
struct sockbuf *sb;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_PEER_CLOSE,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
if (__predict_false(toep->flags & TPF_SYNQE)) {
#ifdef INVARIANTS
struct synq_entry *synqe = (void *)toep;
INP_WLOCK(synqe->lctx->inp);
if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
("%s: listen socket closed but tid %u not aborted.",
__func__, tid));
} else {
/*
* do_pass_accept_req is still running and will
* eventually take care of this tid.
*/
}
INP_WUNLOCK(synqe->lctx->inp);
#endif
CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
toep, toep->flags);
return (0);
}
KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
INP_INFO_WLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = intotcpcb(inp);
CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__,
tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp);
if (toep->flags & TPF_ABORT_SHUTDOWN)
goto done;
tp->rcv_nxt++; /* FIN */
so = inp->inp_socket;
sb = &so->so_rcv;
SOCKBUF_LOCK(sb);
if (__predict_false(toep->ddp_flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) {
m = m_get(M_NOWAIT, MT_DATA);
if (m == NULL)
CXGBE_UNIMPLEMENTED("mbuf alloc failure");
m->m_len = be32toh(cpl->rcv_nxt) - tp->rcv_nxt;
m->m_flags |= M_DDP; /* Data is already where it should be */
m->m_data = "nothing to see here";
tp->rcv_nxt = be32toh(cpl->rcv_nxt);
toep->ddp_flags &= ~(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE);
KASSERT(toep->sb_cc >= sb->sb_cc,
("%s: sb %p has more data (%d) than last time (%d).",
__func__, sb, sb->sb_cc, toep->sb_cc));
toep->rx_credits += toep->sb_cc - sb->sb_cc;
#ifdef USE_DDP_RX_FLOW_CONTROL
toep->rx_credits -= m->m_len; /* adjust for F_RX_FC_DDP */
#endif
sbappendstream_locked(sb, m);
toep->sb_cc = sb->sb_cc;
}
socantrcvmore_locked(so); /* unlocks the sockbuf */
KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),
("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
be32toh(cpl->rcv_nxt)));
switch (tp->t_state) {
case TCPS_SYN_RECEIVED:
tp->t_starttime = ticks;
/* FALLTHROUGH */
case TCPS_ESTABLISHED:
tp->t_state = TCPS_CLOSE_WAIT;
break;
case TCPS_FIN_WAIT_1:
tp->t_state = TCPS_CLOSING;
break;
case TCPS_FIN_WAIT_2:
tcp_twstart(tp);
INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
INP_INFO_WUNLOCK(&V_tcbinfo);
INP_WLOCK(inp);
final_cpl_received(toep);
return (0);
default:
log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
__func__, tid, tp->t_state);
}
done:
INP_WUNLOCK(inp);
INP_INFO_WUNLOCK(&V_tcbinfo);
return (0);
}
/*
* Peer has sent us a FIN.
*/
static int
do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
const struct cpl_peer_close *cpl = (const void *)(rss + 1);
unsigned int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct tcpcb *tp = NULL;
struct socket *so;
#ifdef INVARIANTS
unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl)));
#endif
KASSERT(opcode == CPL_PEER_CLOSE,
("%s: unexpected opcode 0x%x", __func__, opcode));
KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
if (__predict_false(toep->flags & TPF_SYNQE)) {
#ifdef INVARIANTS
struct synq_entry *synqe = (void *)toep;
INP_WLOCK(synqe->lctx->inp);
if (synqe->flags & TPF_SYNQE_HAS_L2TE) {
KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN,
("%s: listen socket closed but tid %u not aborted.",
__func__, tid));
} else {
/*
* do_pass_accept_req is still running and will
* eventually take care of this tid.
*/
}
INP_WUNLOCK(synqe->lctx->inp);
#endif
CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid,
toep, toep->flags);
return (0);
}
KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
INP_INFO_RLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = intotcpcb(inp);
CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__,
tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp);
if (toep->flags & TPF_ABORT_SHUTDOWN)
goto done;
tp->rcv_nxt++; /* FIN */
so = inp->inp_socket;
if (toep->ulp_mode == ULP_MODE_TCPDDP) {
DDP_LOCK(toep);
if (__predict_false(toep->ddp_flags &
(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)))
handle_ddp_close(toep, tp, cpl->rcv_nxt);
DDP_UNLOCK(toep);
}
socantrcvmore(so);
if (toep->ulp_mode != ULP_MODE_RDMA) {
KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),
("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
be32toh(cpl->rcv_nxt)));
}
switch (tp->t_state) {
case TCPS_SYN_RECEIVED:
tp->t_starttime = ticks;
/* FALLTHROUGH */
case TCPS_ESTABLISHED:
tp->t_state = TCPS_CLOSE_WAIT;
break;
case TCPS_FIN_WAIT_1:
tp->t_state = TCPS_CLOSING;
break;
case TCPS_FIN_WAIT_2:
tcp_twstart(tp);
INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */
INP_INFO_RUNLOCK(&V_tcbinfo);
INP_WLOCK(inp);
final_cpl_received(toep);
return (0);
default:
log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n",
__func__, tid, tp->t_state);
}
done:
INP_WUNLOCK(inp);
INP_INFO_RUNLOCK(&V_tcbinfo);
return (0);
}
