static void falcon_rx_unexpected(ef_vi* vi, const ef_vi_event* ev, ef_event** evs, int* evs_len, int q_label, int desc_i) { ef_event* ev_out; if (!((desc_i - 1 - vi->ep_state->rxq.removed) & vi->vi_rxq.mask)) { /* One ahead of expected: previous RX notification lost. */ ev_out = (*evs)++; --(*evs_len); desc_i = (desc_i - 1) & vi->vi_rxq.mask; ev_out->rx_discard.type = EF_EVENT_TYPE_RX_DISCARD; ev_out->rx_discard.subtype = EF_EVENT_RX_DISCARD_EV_ERROR; ev_out->rx_discard.q_id = q_label; ev_out->rx_discard.rq_id = vi->vi_rxq.ids[desc_i]; ev_out->rx_discard.len = 0; vi->vi_rxq.ids[desc_i] = EF_REQUEST_ID_MASK;/* ?? killme */ ++vi->ep_state->rxq.removed; INC_ERROR_STAT(vi, rx_ev_lost); desc_i = (desc_i + 1) & vi->vi_rxq.mask; /* Handle the current event. */ falcon_rx_desc_consumed(vi, ev, evs, evs_len, q_label, desc_i); } else { /* Misdirected? */ INC_ERROR_STAT(vi, rx_ev_bad_desc_i); } }
ef_vi_inline void ef10_rx_event(ef_vi* evq_vi, const ef_vi_event* ev, ef_event** evs, int* evs_len) { unsigned lbits_mask = __EFVI_MASK(ESF_DZ_RX_DSC_PTR_LBITS_WIDTH, unsigned); unsigned q_label = QWORD_GET_U(ESF_DZ_RX_QLABEL, *ev); unsigned short_di, desc_i, q_mask; ef_vi *vi; vi = evq_vi->vi_qs[q_label]; if (likely(vi != NULL)) { q_mask = vi->vi_rxq.mask; short_di = QWORD_GET_U(ESF_DZ_RX_DSC_PTR_LBITS, *ev); desc_i = (vi->ep_state->rxq.removed + ((short_di - vi->ep_state->rxq.removed) & lbits_mask) - 1) & q_mask; huntington_rx_desc_consumed(vi, ev, evs, evs_len, q_label, desc_i); } else { INC_ERROR_STAT(evq_vi, rx_ev_bad_q_label); } }
ef_vi_inline void falcon_rx_event(ef_vi* evq_vi, const ef_vi_event* ev, ef_event** evs, int* evs_len) { unsigned q_label = QWORD_GET_U(RX_EV_Q_LABEL, *ev); unsigned desc_i, q_mask; ef_vi *vi; vi = evq_vi->vi_qs[q_label]; if (likely(vi != NULL)) { q_mask = vi->vi_rxq.mask; desc_i = q_mask & CI_QWORD_FIELD(*ev, RX_EV_DESC_PTR); if (likely(desc_i == (vi->ep_state->rxq.removed & q_mask))) falcon_rx_desc_consumed(vi, ev, evs, evs_len, q_label, desc_i); else if (!((desc_i + 1 - vi->ep_state->rxq.removed) & q_mask)) falcon_rx_no_desc_trunc(evs, evs_len, q_label); else falcon_rx_unexpected(vi, ev, evs, evs_len, q_label, desc_i); } else { INC_ERROR_STAT(evq_vi, rx_ev_bad_q_label); } }
int falcon_ef_eventq_poll(ef_vi* evq, ef_event* evs, int evs_len) { int evs_len_orig = evs_len; ef_vi_event *pev, ev; EF_VI_BUG_ON(evs == NULL); EF_VI_BUG_ON(evs_len < EF_VI_EVENT_POLL_MIN_EVS); #ifdef __powerpc__ if(unlikely( EF_VI_IS_EVENT(EF_VI_EVENT_PTR(evq, -17)) )) goto overflow; #else if(unlikely( EF_VI_IS_EVENT(EF_VI_EVENT_PTR(evq, -1)) )) goto overflow; #endif not_empty: /* Read the event out of the ring, then fiddle with copied version. * Reason is that the ring is likely to get pushed out of cache by * another event being delivered by hardware. */ pev = EF_VI_EVENT_PTR(evq, 0); ev = *pev; if (!EF_VI_IS_EVENT(&ev)) goto empty; do { #ifdef __powerpc__ CI_SET_QWORD(*EF_VI_EVENT_PTR(evq, -16)); #else CI_SET_QWORD(*pev); #endif evq->ep_state->evq.evq_ptr += sizeof(ef_vi_event); /* Ugly: Exploit the fact that event code lies in top bits * of event. */ EF_VI_BUG_ON(EV_CODE_LBN < 32u); switch( CI_QWORD_FIELD(ev, EV_CODE) ) { case RX_IP_EV_DECODE: falcon_rx_event(evq, &ev, &evs, &evs_len); break; case TX_IP_EV_DECODE: falcon_tx_event(evs, &ev); --evs_len; ++evs; break; case DRV_GEN_EV_DECODE: falcon_drv_gen_event(evq, &ev, &evs, &evs_len); break; default: break; } if (evs_len == 0) break; pev = EF_VI_EVENT_PTR(evq, 0); ev = *pev; } while (EF_VI_IS_EVENT(&ev)); return evs_len_orig - evs_len; empty: if (EF_VI_IS_EVENT(EF_VI_EVENT_PTR(evq, 1))) { smp_rmb(); if (!EF_VI_IS_EVENT(EF_VI_EVENT_PTR(evq, 0))) { /* No event in current slot, but there is one in * the next slot. Has NIC failed to write event * somehow? */ evq->ep_state->evq.evq_ptr += sizeof(ef_vi_event); INC_ERROR_STAT(evq, evq_gap); goto not_empty; } } return 0; overflow: evs->generic.type = EF_EVENT_TYPE_OFLOW; return 1; }