ef_vi_inline void ef10_packed_stream_rx_event(ef_vi* evq_vi,
const ef_vi_event* ev,
ef_event** evs, int* evs_len)
{
unsigned q_label = QWORD_GET_U(ESF_DZ_RX_QLABEL, *ev);
unsigned short_pc = QWORD_GET_U(ESF_DZ_RX_DSC_PTR_LBITS, *ev);
unsigned pkt_count_range = (1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH);
const ci_uint32 discard_mask =
CI_BSWAPC_LE32(1 << ESF_DZ_RX_ECC_ERR_LBN |
1 << ESF_DZ_RX_CRC1_ERR_LBN |
1 << ESF_DZ_RX_CRC0_ERR_LBN |
1 << ESF_DZ_RX_TCPUDP_CKSUM_ERR_LBN |
1 << ESF_DZ_RX_IPCKSUM_ERR_LBN |
1 << ESF_DZ_RX_ECRC_ERR_LBN);
ef_vi* vi = evq_vi->vi_qs[q_label];
ef_event* ev_out = (*evs)++;
--(*evs_len);
ev_out->rx_packed_stream.type = EF_EVENT_TYPE_RX_PACKED_STREAM;
ev_out->rx_packed_stream.q_id = q_label;
ev_out->rx_packed_stream.n_pkts =
(pkt_count_range + short_pc -
vi->ep_state->rxq.rx_ps_pkt_count) % pkt_count_range;
ev_out->rx_packed_stream.flags = 0;
ev_out->rx_packed_stream.ps_flags = 0;
vi->ep_state->rxq.rx_ps_pkt_count = short_pc;
if (unlikely( QWORD_GET_U(ESF_DZ_RX_EV_ROTATE, *ev) )) {
unsigned desc_id;
desc_id = evq_vi->ep_state->rxq.removed & vi->vi_rxq.mask;
vi->vi_rxq.ids[desc_id] = EF_REQUEST_ID_MASK;
++evq_vi->ep_state->rxq.removed;
EF_VI_ASSERT(vi->ep_state->rxq.rx_ps_credit_avail > 0);
--vi->ep_state->rxq.rx_ps_credit_avail;
ev_out->rx_packed_stream.flags |= EF_EVENT_FLAG_PS_NEXT_BUFFER;
}
EF_VI_ASSERT(ev_out->rx_packed_stream.n_pkts <= 8);
EF_VI_ASSERT(ev_out->rx_packed_stream.n_pkts > 0 ||
QWORD_GET_U(ESF_DZ_RX_CONT, *ev));
if (likely( ! ((ev->u32[0] & discard_mask) )))
return;
if (QWORD_GET_U(ESF_DZ_RX_ECC_ERR, *ev) |
QWORD_GET_U(ESF_DZ_RX_CRC1_ERR, *ev) |
QWORD_GET_U(ESF_DZ_RX_CRC0_ERR, *ev) |
QWORD_GET_U(ESF_DZ_RX_ECRC_ERR, *ev))
ev_out->rx_packed_stream.ps_flags |= EF_VI_PS_FLAG_BAD_FCS;
if (QWORD_GET_U(ESF_DZ_RX_TCPUDP_CKSUM_ERR, *ev))
ev_out->rx_packed_stream.ps_flags |= EF_VI_PS_FLAG_BAD_L4_CSUM;
if (QWORD_GET_U(ESF_DZ_RX_IPCKSUM_ERR, *ev))
ev_out->rx_packed_stream.ps_flags |= EF_VI_PS_FLAG_BAD_L3_CSUM;
}
int ef_vi_receive_unbundle(ef_vi* vi, const ef_event* ev,
ef_request_id* ids)
{
ef_request_id* ids_in = ids;
ef_vi_rxq* q = &vi->vi_rxq;
ef_vi_rxq_state* qs = &vi->ep_state->rxq;
unsigned i;
EF_VI_BUG_ON( EF_EVENT_TYPE(*ev) != EF_EVENT_TYPE_RX_MULTI &&
EF_EVENT_TYPE(*ev) != EF_EVENT_TYPE_RX_MULTI_DISCARD );
EF_VI_BUG_ON( ev->rx_multi.n_descs > EF_VI_RECEIVE_BATCH );
for( i = 0; i < ev->rx_multi.n_descs; ++i ) {
unsigned di = qs->removed & q->mask;
++(qs->removed);
if( q->ids[di] != EF_REQUEST_ID_MASK ) {
*ids++ = q->ids[di];
q->ids[di] = EF_REQUEST_ID_MASK;
}
}
/* Check we didn't remove more than we've added. */
EF_VI_ASSERT( qs->added - qs->removed <= q->mask );
return (int) (ids - ids_in);
}
static void ef10_tx_event_ts_enabled(ef_vi* evq, const ef_vi_event* ev,
ef_event** evs, int* evs_len)
{
EF_VI_ASSERT(evq->vi_flags & EF_VI_TX_TIMESTAMPS);
/* When TX timestamping is enabled, we get three events for
* every transmit. A TX completion and two timestamp events.
* We ignore the completion and store the first timestamp in
* the per TXQ state. On the second timestamp we retrieve the
* first one and construct a EF_EVENT_TYPE_TX_WITH_TIMESTAMP
* event to send to the user. */
if(QWORD_GET_U(ESF_DZ_TX_SOFT1, *ev) ==
TX_TIMESTAMP_EVENT_TX_EV_COMPLETION) {
/* TX completion event. Ignored */
}
else if(QWORD_GET_U(ESF_DZ_TX_SOFT1, *ev) ==
TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_LO) {
ef_vi_txq_state* qs = &evq->ep_state->txq;
EF_VI_DEBUG(
EF_VI_BUG_ON(qs->ts_nsec !=
EF_VI_TX_TIMESTAMP_TS_NSEC_INVALID)
);
qs->ts_nsec =
(((((uint64_t)timestamp_extract(*ev)) *
1000000000UL) >> 29) << 2) |
evq->ep_state->evq.sync_flags;
}
void falcon_ef_eventq_prime(ef_vi* vi)
{
unsigned ring_i = (ef_eventq_current(vi) & vi->evq_mask) / 8;
void* io = vi->io + (FR_BZ_EVQ_RPTR_REGP0_OFST & (EF_VI_PAGE_SIZE-1));
EF_VI_ASSERT(vi->inited & EF_VI_INITED_IO);
writel(ring_i << FRF_AZ_EVQ_RPTR_LBN, io);
mmiowb();
}
int ef_eventq_has_event(ef_vi* vi)
{
EF_VI_ASSERT(vi->evq_base);
return EF_VI_IS_EVENT(EF_VI_EVENT_PTR(vi, 0));
}
