static int
do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cxgbei_data *ci = sc->iscsi_ulp_softc;
struct cpl_iscsi_data *cpl = mtod(m, struct cpl_iscsi_data *);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct icl_cxgbei_pdu *icp = toep->ulpcb2;
M_ASSERTPKTHDR(m);
MPASS(m->m_pkthdr.len == be16toh(cpl->len) + sizeof(*cpl));
/* Must already have received the header (but not the data). */
MPASS(icp != NULL);
MPASS(icp->icp_flags == ICPF_RX_HDR);
MPASS(icp->ip.ip_data_mbuf == NULL);
m_adj(m, sizeof(*cpl));
MPASS(icp->ip.ip_data_len == m->m_pkthdr.len);
icp->icp_flags |= ICPF_RX_FLBUF;
icp->ip.ip_data_mbuf = m;
counter_u64_add(ci->fl_pdus, 1);
counter_u64_add(ci->fl_bytes, m->m_pkthdr.len);
#if 0
CTR3(KTR_CXGBE, "%s: tid %u, cpl->len %u", __func__, tid,
be16toh(cpl->len));
#endif
return (0);
}
static struct mbuf *
rtwn_report_intr(struct rtwn_usb_softc *uc, struct usb_xfer *xfer,
struct rtwn_data *data)
{
struct rtwn_softc *sc = &uc->uc_sc;
struct ieee80211com *ic = &sc->sc_ic;
uint8_t *buf;
int len;
usbd_xfer_status(xfer, &len, NULL, NULL, NULL);
if (__predict_false(len < sizeof(struct r92c_rx_stat))) {
counter_u64_add(ic->ic_ierrors, 1);
return (NULL);
}
buf = data->buf;
switch (rtwn_classify_intr(sc, buf, len)) {
case RTWN_RX_DATA:
return (rtwn_rxeof(sc, buf, len));
case RTWN_RX_TX_REPORT:
if (sc->sc_ratectl != RTWN_RATECTL_NET80211) {
/* shouldn't happen */
device_printf(sc->sc_dev,
"%s called while ratectl = %d!\n",
__func__, sc->sc_ratectl);
break;
}
RTWN_NT_LOCK(sc);
rtwn_handle_tx_report(sc, buf, len);
RTWN_NT_UNLOCK(sc);
#ifdef IEEE80211_SUPPORT_SUPERG
/*
* NB: this will executed only when 'report' bit is set.
*/
if (sc->sc_tx_n_active > 0 && --sc->sc_tx_n_active <= 1)
rtwn_cmd_sleepable(sc, NULL, 0, rtwn_ff_flush_all);
#endif
break;
case RTWN_RX_OTHER:
rtwn_handle_c2h_report(sc, buf, len);
break;
default:
/* NOTREACHED */
KASSERT(0, ("unknown Rx classification code"));
break;
}
return (NULL);
}
/*
* MP-friendly version of ppsratecheck().
*
* Returns non-negative if we are in the rate, negative otherwise.
* 0 - rate limit not reached.
* -1 - rate limit reached.
* >0 - rate limit was reached before, and was just reset. The return value
* is number of events since last reset.
*/
int64_t
counter_ratecheck(struct counter_rate *cr, int64_t limit)
{
int64_t val;
int now;
val = cr->cr_over;
now = ticks;
if (now - cr->cr_ticks >= hz) {
/*
* Time to clear the structure, we are in the next second.
* First try unlocked read, and then proceed with atomic.
*/
if ((cr->cr_lock == 0) &&
atomic_cmpset_acq_int(&cr->cr_lock, 0, 1)) {
/*
* Check if other thread has just went through the
* reset sequence before us.
*/
if (now - cr->cr_ticks >= hz) {
val = counter_u64_fetch(cr->cr_rate);
counter_u64_zero(cr->cr_rate);
cr->cr_over = 0;
cr->cr_ticks = now;
if (val <= limit)
val = 0;
}
atomic_store_rel_int(&cr->cr_lock, 0);
} else
/*
* We failed to lock, in this case other thread may
* be running counter_u64_zero(), so it is not safe
* to do an update, we skip it.
*/
return (val);
}
counter_u64_add(cr->cr_rate, 1);
if (cr->cr_over != 0)
return (-1);
if (counter_u64_fetch(cr->cr_rate) > limit)
val = cr->cr_over = -1;
return (val);
}
/*
* Enqueue n items and maybe drain the ring for some time.
*
* Returns an errno.
*/
int
mp_ring_enqueue(struct mp_ring *r, void **items, int n, int budget)
{
union ring_state os, ns;
uint16_t pidx_start, pidx_stop;
int i;
MPASS(items != NULL);
MPASS(n > 0);
/*
* Reserve room for the new items. Our reservation, if successful, is
* from 'pidx_start' to 'pidx_stop'.
*/
for (;;) {
os.state = r->state;
if (n >= space_available(r, os)) {
counter_u64_add(r->drops, n);
MPASS(os.flags != IDLE);
if (os.flags == STALLED)
mp_ring_check_drainage(r, 0);
return (ENOBUFS);
}
ns.state = os.state;
ns.pidx_head = increment_idx(r, os.pidx_head, n);
critical_enter();
if (atomic_cmpset_64(&r->state, os.state, ns.state))
break;
critical_exit();
cpu_spinwait();
}
pidx_start = os.pidx_head;
pidx_stop = ns.pidx_head;
/*
* Wait for other producers who got in ahead of us to enqueue their
* items, one producer at a time. It is our turn when the ring's
* pidx_tail reaches the begining of our reservation (pidx_start).
*/
while (ns.pidx_tail != pidx_start) {
cpu_spinwait();
ns.state = r->state;
}
/* Now it is our turn to fill up the area we reserved earlier. */
i = pidx_start;
do {
r->items[i] = *items++;
if (__predict_false(++i == r->size))
i = 0;
} while (i != pidx_stop);
/*
* Update the ring's pidx_tail. The release style atomic guarantees
* that the items are visible to any thread that sees the updated pidx.
*/
do {
os.state = ns.state = r->state;
ns.pidx_tail = pidx_stop;
ns.flags = BUSY;
} while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0);
critical_exit();
counter_u64_add(r->enqueues, n);
/*
* Turn into a consumer if some other thread isn't active as a consumer
* already.
*/
if (os.flags != BUSY)
drain_ring(r, ns, os.flags, budget);
return (0);
}
/*
* Caller passes in a state, with a guarantee that there is work to do and that
* all items up to the pidx_tail in the state are visible.
*/
static void
drain_ring(struct mp_ring *r, union ring_state os, uint16_t prev, int budget)
{
union ring_state ns;
int n, pending, total;
uint16_t cidx = os.cidx;
uint16_t pidx = os.pidx_tail;
MPASS(os.flags == BUSY);
MPASS(cidx != pidx);
if (prev == IDLE)
counter_u64_add(r->starts, 1);
pending = 0;
total = 0;
while (cidx != pidx) {
/* Items from cidx to pidx are available for consumption. */
n = r->drain(r, cidx, pidx);
if (n == 0) {
critical_enter();
do {
os.state = ns.state = r->state;
ns.cidx = cidx;
ns.flags = STALLED;
} while (atomic_cmpset_64(&r->state, os.state,
ns.state) == 0);
critical_exit();
if (prev != STALLED)
counter_u64_add(r->stalls, 1);
else if (total > 0) {
counter_u64_add(r->restarts, 1);
counter_u64_add(r->stalls, 1);
}
break;
}
cidx = increment_idx(r, cidx, n);
pending += n;
total += n;
/*
* We update the cidx only if we've caught up with the pidx, the
* real cidx is getting too far ahead of the one visible to
* everyone else, or we have exceeded our budget.
*/
if (cidx != pidx && pending < 64 && total < budget)
continue;
critical_enter();
do {
os.state = ns.state = r->state;
ns.cidx = cidx;
ns.flags = state_to_flags(ns, total >= budget);
} while (atomic_cmpset_acq_64(&r->state, os.state, ns.state) == 0);
critical_exit();
if (ns.flags == ABDICATED)
counter_u64_add(r->abdications, 1);
if (ns.flags != BUSY) {
/* Wrong loop exit if we're going to stall. */
MPASS(ns.flags != STALLED);
if (prev == STALLED) {
MPASS(total > 0);
counter_u64_add(r->restarts, 1);
}
break;
}
/*
* The acquire style atomic above guarantees visibility of items
* associated with any pidx change that we notice here.
*/
pidx = ns.pidx_tail;
pending = 0;
}
}
/*
* Kernel module interface for updating udpstat. The argument is an index
* into udpstat treated as an array of u_long. While this encodes the
* general layout of udpstat into the caller, it doesn't encode its location,
* so that future changes to add, for example, per-CPU stats support won't
* cause binary compatibility problems for kernel modules.
*/
void
kmod_udpstat_inc(int statnum)
{
counter_u64_add(VNET(udpstat)[statnum], 1);
}
static int
do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
struct adapter *sc = iq->adapter;
struct cxgbei_data *ci = sc->iscsi_ulp_softc;
const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
u_int tid = GET_TID(cpl);
struct toepcb *toep = lookup_tid(sc, tid);
struct inpcb *inp = toep->inp;
struct socket *so;
struct sockbuf *sb;
struct tcpcb *tp;
struct icl_cxgbei_conn *icc;
struct icl_conn *ic;
struct icl_cxgbei_pdu *icp = toep->ulpcb2;
struct icl_pdu *ip;
u_int pdu_len, val;
MPASS(m == NULL);
/* Must already be assembling a PDU. */
MPASS(icp != NULL);
MPASS(icp->icp_flags & ICPF_RX_HDR); /* Data is optional. */
MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0);
pdu_len = be16toh(cpl->len); /* includes everything. */
val = be32toh(cpl->ddpvld);
#if 0
CTR5(KTR_CXGBE,
"%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp_flags 0x%08x",
__func__, tid, pdu_len, val, icp->icp_flags);
#endif
icp->icp_flags |= ICPF_RX_STATUS;
ip = &icp->ip;
if (val & F_DDP_PADDING_ERR)
icp->icp_flags |= ICPF_PAD_ERR;
if (val & F_DDP_HDRCRC_ERR)
icp->icp_flags |= ICPF_HCRC_ERR;
if (val & F_DDP_DATACRC_ERR)
icp->icp_flags |= ICPF_DCRC_ERR;
if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) {
MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0);
MPASS(ip->ip_data_len > 0);
icp->icp_flags |= ICPF_RX_DDP;
counter_u64_add(ci->ddp_pdus, 1);
counter_u64_add(ci->ddp_bytes, ip->ip_data_len);
}
INP_WLOCK(inp);
if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
__func__, tid, pdu_len, inp->inp_flags);
INP_WUNLOCK(inp);
icl_cxgbei_conn_pdu_free(NULL, ip);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
tp = intotcpcb(inp);
MPASS(icp->icp_seq == tp->rcv_nxt);
MPASS(tp->rcv_wnd >= pdu_len);
tp->rcv_nxt += pdu_len;
tp->rcv_wnd -= pdu_len;
tp->t_rcvtime = ticks;
/* update rx credits */
toep->rx_credits += pdu_len;
t4_rcvd(&toep->td->tod, tp); /* XXX: sc->tom_softc.tod */
so = inp->inp_socket;
sb = &so->so_rcv;
SOCKBUF_LOCK(sb);
icc = toep->ulpcb;
if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) {
CTR5(KTR_CXGBE,
"%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
__func__, tid, pdu_len, icc, sb->sb_state);
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
INP_INFO_RLOCK(&V_tcbinfo);
INP_WLOCK(inp);
tp = tcp_drop(tp, ECONNRESET);
if (tp)
INP_WUNLOCK(inp);
INP_INFO_RUNLOCK(&V_tcbinfo);
icl_cxgbei_conn_pdu_free(NULL, ip);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
ic = &icc->ic;
icl_cxgbei_new_pdu_set_conn(ip, ic);
MPASS(m == NULL); /* was unused, we'll use it now. */
m = sbcut_locked(sb, sbused(sb)); /* XXXNP: toep->sb_cc accounting? */
if (__predict_false(m != NULL)) {
int len = m_length(m, NULL);
/*
* PDUs were received before the tid transitioned to ULP mode.
* Convert them to icl_cxgbei_pdus and send them to ICL before
* the PDU in icp/ip.
*/
CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, tid,
len);
/* XXXNP: needs to be rewritten. */
if (len == sizeof(struct iscsi_bhs) || len == 4 + sizeof(struct
iscsi_bhs)) {
struct icl_cxgbei_pdu *icp0;
struct icl_pdu *ip0;
ip0 = icl_cxgbei_new_pdu(M_NOWAIT);
icl_cxgbei_new_pdu_set_conn(ip0, ic);
if (ip0 == NULL)
CXGBE_UNIMPLEMENTED("PDU allocation failure");
icp0 = ip_to_icp(ip0);
icp0->icp_seq = 0; /* XXX */
icp0->icp_flags = ICPF_RX_HDR | ICPF_RX_STATUS;
m_copydata(m, 0, sizeof(struct iscsi_bhs), (void *)ip0->ip_bhs);
STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip0, ip_next);
}
m_freem(m);
}
STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
if ((icc->rx_flags & RXF_ACTIVE) == 0) {
struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt];
mtx_lock(&cwt->cwt_lock);
icc->rx_flags |= RXF_ACTIVE;
TAILQ_INSERT_TAIL(&cwt->rx_head, icc, rx_link);
if (cwt->cwt_state == CWT_SLEEPING) {
cwt->cwt_state = CWT_RUNNING;
cv_signal(&cwt->cwt_cv);
}
mtx_unlock(&cwt->cwt_lock);
}
SOCKBUF_UNLOCK(sb);
INP_WUNLOCK(inp);
#ifdef INVARIANTS
toep->ulpcb2 = NULL;
#endif
return (0);
}
static int
mpc7xxx_intr(struct trapframe *tf)
{
int i, error, retval, cpu;
uint32_t config;
struct pmc *pm;
struct powerpc_cpu *pac;
cpu = curcpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] out of range CPU %d", __LINE__, cpu));
PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
retval = 0;
pac = powerpc_pcpu[cpu];
config = mfspr(SPR_MMCR0) & ~SPR_MMCR0_FC;
/*
* look for all PMCs that have interrupted:
* - look for a running, sampling PMC which has overflowed
* and which has a valid 'struct pmc' association
*
* If found, we call a helper to process the interrupt.
*/
for (i = 0; i < MPC7XXX_MAX_PMCS; i++) {
if ((pm = pac->pc_ppcpmcs[i].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
if (!MPC7XXX_PMC_HAS_OVERFLOWED(i))
continue;
retval = 1; /* Found an interrupting PMC. */
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
/* Stop the counter if logging fails. */
error = pmc_process_interrupt(PMC_HR, pm, tf);
if (error != 0)
mpc7xxx_stop_pmc(cpu, i);
/* reload count. */
mpc7xxx_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
}
if (retval)
counter_u64_add(pmc_stats.pm_intr_processed, 1);
else
counter_u64_add(pmc_stats.pm_intr_ignored, 1);
/* Re-enable PERF exceptions. */
if (retval)
mtspr(SPR_MMCR0, config | SPR_MMCR0_PMXE);
return (retval);
}
static struct mbuf *
rtwn_rx_copy_to_mbuf(struct rtwn_softc *sc, struct r92c_rx_stat *stat,
int totlen)
{
struct ieee80211com *ic = &sc->sc_ic;
struct mbuf *m;
uint32_t rxdw0;
int pktlen;
RTWN_ASSERT_LOCKED(sc);
/* Dump Rx descriptor. */
RTWN_DPRINTF(sc, RTWN_DEBUG_RECV_DESC,
"%s: dw: 0 %08X, 1 %08X, 2 %08X, 3 %08X, 4 %08X, tsfl %08X\n",
__func__, le32toh(stat->rxdw0), le32toh(stat->rxdw1),
le32toh(stat->rxdw2), le32toh(stat->rxdw3), le32toh(stat->rxdw4),
le32toh(stat->tsf_low));
/*
* don't pass packets to the ieee80211 framework if the driver isn't
* RUNNING.
*/
if (!(sc->sc_flags & RTWN_RUNNING))
return (NULL);
rxdw0 = le32toh(stat->rxdw0);
if (__predict_false(rxdw0 & (R92C_RXDW0_CRCERR | R92C_RXDW0_ICVERR))) {
/*
* This should not happen since we setup our Rx filter
* to not receive these frames.
*/
RTWN_DPRINTF(sc, RTWN_DEBUG_RECV,
"%s: RX flags error (%s)\n", __func__,
rxdw0 & R92C_RXDW0_CRCERR ? "CRC" : "ICV");
goto fail;
}
pktlen = MS(rxdw0, R92C_RXDW0_PKTLEN);
if (__predict_false(pktlen < sizeof(struct ieee80211_frame_ack))) {
/*
* Should not happen (because of Rx filter setup).
*/
RTWN_DPRINTF(sc, RTWN_DEBUG_RECV,
"%s: frame is too short: %d\n", __func__, pktlen);
goto fail;
}
m = m_get2(totlen, M_NOWAIT, MT_DATA, M_PKTHDR);
if (__predict_false(m == NULL)) {
device_printf(sc->sc_dev, "%s: could not allocate RX mbuf\n",
__func__);
goto fail;
}
/* Finalize mbuf. */
memcpy(mtod(m, uint8_t *), (uint8_t *)stat, totlen);
m->m_pkthdr.len = m->m_len = totlen;
if (rtwn_check_frame(sc, m) != 0) {
m_freem(m);
goto fail;
}
return (m);
fail:
counter_u64_add(ic->ic_ierrors, 1);
return (NULL);
}
void
rtwn_bulk_rx_callback(struct usb_xfer *xfer, usb_error_t error)
{
struct rtwn_usb_softc *uc = usbd_xfer_softc(xfer);
struct rtwn_softc *sc = &uc->uc_sc;
struct ieee80211com *ic = &sc->sc_ic;
struct ieee80211_node *ni;
struct mbuf *m = NULL, *next;
struct rtwn_data *data;
int8_t nf, rssi;
RTWN_ASSERT_LOCKED(sc);
switch (USB_GET_STATE(xfer)) {
case USB_ST_TRANSFERRED:
data = STAILQ_FIRST(&uc->uc_rx_active);
if (data == NULL)
goto tr_setup;
STAILQ_REMOVE_HEAD(&uc->uc_rx_active, next);
m = rtwn_report_intr(uc, xfer, data);
STAILQ_INSERT_TAIL(&uc->uc_rx_inactive, data, next);
/* FALLTHROUGH */
case USB_ST_SETUP:
tr_setup:
data = STAILQ_FIRST(&uc->uc_rx_inactive);
if (data == NULL) {
KASSERT(m == NULL, ("mbuf isn't NULL"));
goto finish;
}
STAILQ_REMOVE_HEAD(&uc->uc_rx_inactive, next);
STAILQ_INSERT_TAIL(&uc->uc_rx_active, data, next);
usbd_xfer_set_frame_data(xfer, 0, data->buf,
usbd_xfer_max_len(xfer));
usbd_transfer_submit(xfer);
/*
* To avoid LOR we should unlock our private mutex here to call
* ieee80211_input() because here is at the end of a USB
* callback and safe to unlock.
*/
while (m != NULL) {
next = m->m_next;
m->m_next = NULL;
ni = rtwn_rx_frame(sc, m, &rssi);
RTWN_UNLOCK(sc);
nf = RTWN_NOISE_FLOOR;
if (ni != NULL) {
if (ni->ni_flags & IEEE80211_NODE_HT)
m->m_flags |= M_AMPDU;
(void)ieee80211_input(ni, m, rssi - nf, nf);
ieee80211_free_node(ni);
} else {
(void)ieee80211_input_all(ic, m,
rssi - nf, nf);
}
RTWN_LOCK(sc);
m = next;
}
break;
default:
/* needs it to the inactive queue due to a error. */
data = STAILQ_FIRST(&uc->uc_rx_active);
if (data != NULL) {
STAILQ_REMOVE_HEAD(&uc->uc_rx_active, next);
STAILQ_INSERT_TAIL(&uc->uc_rx_inactive, data, next);
}
if (error != USB_ERR_CANCELLED) {
usbd_xfer_set_stall(xfer);
counter_u64_add(ic->ic_ierrors, 1);
goto tr_setup;
}
break;
}
finish:
/* Finished receive; age anything left on the FF queue by a little bump */
/*
* XXX TODO: just make this a callout timer schedule so we can
* flush the FF staging queue if we're approaching idle.
*/
#ifdef IEEE80211_SUPPORT_SUPERG
if (!(sc->sc_flags & RTWN_FW_LOADED) ||
sc->sc_ratectl != RTWN_RATECTL_NET80211)
rtwn_cmd_sleepable(sc, NULL, 0, rtwn_ff_flush_all);
#endif
/* Kick-start more transmit in case we stalled */
rtwn_start(sc);
}