static void falcon_rx_unexpected(ef_vi* vi, const ef_vi_event* ev,
ef_event** evs, int* evs_len,
int q_label, int desc_i)
{
ef_event* ev_out;
if (!((desc_i - 1 - vi->ep_state->rxq.removed) & vi->vi_rxq.mask)) {
/* One ahead of expected: previous RX notification lost. */
ev_out = (*evs)++;
--(*evs_len);
desc_i = (desc_i - 1) & vi->vi_rxq.mask;
ev_out->rx_discard.type = EF_EVENT_TYPE_RX_DISCARD;
ev_out->rx_discard.subtype = EF_EVENT_RX_DISCARD_EV_ERROR;
ev_out->rx_discard.q_id = q_label;
ev_out->rx_discard.rq_id = vi->vi_rxq.ids[desc_i];
ev_out->rx_discard.len = 0;
vi->vi_rxq.ids[desc_i] = EF_REQUEST_ID_MASK;/* ?? killme */
++vi->ep_state->rxq.removed;
INC_ERROR_STAT(vi, rx_ev_lost);
desc_i = (desc_i + 1) & vi->vi_rxq.mask;
/* Handle the current event. */
falcon_rx_desc_consumed(vi, ev, evs, evs_len, q_label, desc_i);
} else {
/* Misdirected? */
INC_ERROR_STAT(vi, rx_ev_bad_desc_i);
}
}
ef_vi_inline void ef10_rx_event(ef_vi* evq_vi, const ef_vi_event* ev,
ef_event** evs, int* evs_len)
{
unsigned lbits_mask = __EFVI_MASK(ESF_DZ_RX_DSC_PTR_LBITS_WIDTH,
unsigned);
unsigned q_label = QWORD_GET_U(ESF_DZ_RX_QLABEL, *ev);
unsigned short_di, desc_i, q_mask;
ef_vi *vi;
vi = evq_vi->vi_qs[q_label];
if (likely(vi != NULL)) {
q_mask = vi->vi_rxq.mask;
short_di = QWORD_GET_U(ESF_DZ_RX_DSC_PTR_LBITS, *ev);
desc_i = (vi->ep_state->rxq.removed +
((short_di - vi->ep_state->rxq.removed) &
lbits_mask) - 1) & q_mask;
huntington_rx_desc_consumed(vi, ev, evs, evs_len,
q_label, desc_i);
} else {
INC_ERROR_STAT(evq_vi, rx_ev_bad_q_label);
}
}
ef_vi_inline void falcon_rx_event(ef_vi* evq_vi, const ef_vi_event* ev,
ef_event** evs, int* evs_len)
{
unsigned q_label = QWORD_GET_U(RX_EV_Q_LABEL, *ev);
unsigned desc_i, q_mask;
ef_vi *vi;
vi = evq_vi->vi_qs[q_label];
if (likely(vi != NULL)) {
q_mask = vi->vi_rxq.mask;
desc_i = q_mask & CI_QWORD_FIELD(*ev, RX_EV_DESC_PTR);
if (likely(desc_i == (vi->ep_state->rxq.removed & q_mask)))
falcon_rx_desc_consumed(vi, ev, evs, evs_len,
q_label, desc_i);
else if (!((desc_i + 1 - vi->ep_state->rxq.removed) & q_mask))
falcon_rx_no_desc_trunc(evs, evs_len, q_label);
else
falcon_rx_unexpected(vi, ev, evs, evs_len,
q_label, desc_i);
} else {
INC_ERROR_STAT(evq_vi, rx_ev_bad_q_label);
}
}
int falcon_ef_eventq_poll(ef_vi* evq, ef_event* evs, int evs_len)
{
int evs_len_orig = evs_len;
ef_vi_event *pev, ev;
EF_VI_BUG_ON(evs == NULL);
EF_VI_BUG_ON(evs_len < EF_VI_EVENT_POLL_MIN_EVS);
#ifdef __powerpc__
if(unlikely( EF_VI_IS_EVENT(EF_VI_EVENT_PTR(evq, -17)) ))
goto overflow;
#else
if(unlikely( EF_VI_IS_EVENT(EF_VI_EVENT_PTR(evq, -1)) ))
goto overflow;
#endif
not_empty:
/* Read the event out of the ring, then fiddle with copied version.
* Reason is that the ring is likely to get pushed out of cache by
* another event being delivered by hardware.
*/
pev = EF_VI_EVENT_PTR(evq, 0);
ev = *pev;
if (!EF_VI_IS_EVENT(&ev))
goto empty;
do {
#ifdef __powerpc__
CI_SET_QWORD(*EF_VI_EVENT_PTR(evq, -16));
#else
CI_SET_QWORD(*pev);
#endif
evq->ep_state->evq.evq_ptr += sizeof(ef_vi_event);
/* Ugly: Exploit the fact that event code lies in top bits
* of event. */
EF_VI_BUG_ON(EV_CODE_LBN < 32u);
switch( CI_QWORD_FIELD(ev, EV_CODE) ) {
case RX_IP_EV_DECODE:
falcon_rx_event(evq, &ev, &evs, &evs_len);
break;
case TX_IP_EV_DECODE:
falcon_tx_event(evs, &ev);
--evs_len;
++evs;
break;
case DRV_GEN_EV_DECODE:
falcon_drv_gen_event(evq, &ev, &evs, &evs_len);
break;
default:
break;
}
if (evs_len == 0)
break;
pev = EF_VI_EVENT_PTR(evq, 0);
ev = *pev;
} while (EF_VI_IS_EVENT(&ev));
return evs_len_orig - evs_len;
empty:
if (EF_VI_IS_EVENT(EF_VI_EVENT_PTR(evq, 1))) {
smp_rmb();
if (!EF_VI_IS_EVENT(EF_VI_EVENT_PTR(evq, 0))) {
/* No event in current slot, but there is one in
* the next slot. Has NIC failed to write event
* somehow?
*/
evq->ep_state->evq.evq_ptr += sizeof(ef_vi_event);
INC_ERROR_STAT(evq, evq_gap);
goto not_empty;
}
}
return 0;
overflow:
evs->generic.type = EF_EVENT_TYPE_OFLOW;
return 1;
}
