static boolean_t
sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
uint16_t flags)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
struct sfxge_rxq *rxq;
unsigned int expected;
struct sfxge_rx_sw_desc *rx_desc;
evq = arg;
sc = evq->sc;
if (evq->exception)
goto done;
rxq = sc->rxq[label];
KASSERT(rxq != NULL, ("rxq == NULL"));
KASSERT(evq->index == rxq->index,
("evq->index != rxq->index"));
if (rxq->init_state != SFXGE_RXQ_STARTED)
goto done;
expected = rxq->pending++ & (SFXGE_NDESCS - 1);
if (id != expected) {
evq->exception = B_TRUE;
device_printf(sc->dev, "RX completion out of order"
" (id=%#x expected=%#x flags=%#x); resetting\n",
id, expected, flags);
sfxge_schedule_reset(sc);
goto done;
}
rx_desc = &rxq->queue[id];
KASSERT(rx_desc->flags == EFX_DISCARD,
("rx_desc->flags != EFX_DISCARD"));
rx_desc->flags = flags;
KASSERT(size < (1 << 16), ("size > (1 << 16)"));
rx_desc->size = (uint16_t)size;
prefetch_read_many(rx_desc->mbuf);
evq->rx_done++;
if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
sfxge_ev_qcomplete(evq, B_FALSE);
done:
return (evq->rx_done >= SFXGE_EV_BATCH);
}
int
sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index)
{
struct sfxge_evq *evq;
int rc;
evq = sc->evq[index];
mtx_lock(&evq->lock);
if (evq->init_state != SFXGE_EVQ_STARTING &&
evq->init_state != SFXGE_EVQ_STARTED) {
rc = EINVAL;
goto fail;
}
/* Synchronize the DMA memory for reading */
bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
BUS_DMASYNC_POSTREAD);
KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
/* Poll the queue */
efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
evq->rx_done = 0;
evq->tx_done = 0;
/* Perform any pending completion processing */
sfxge_ev_qcomplete(evq, B_TRUE);
/* Re-prime the event queue for interrupts */
if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
goto fail;
mtx_unlock(&evq->lock);
return (0);
fail:
mtx_unlock(&(evq->lock));
return (rc);
}
int
sfxge_ev_qpoll(struct sfxge_evq *evq)
{
int rc;
SFXGE_EVQ_LOCK(evq);
if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
evq->init_state != SFXGE_EVQ_STARTED)) {
rc = EINVAL;
goto fail;
}
/* Synchronize the DMA memory for reading */
bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
BUS_DMASYNC_POSTREAD);
KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
/* Poll the queue */
efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
evq->rx_done = 0;
evq->tx_done = 0;
/* Perform any pending completion processing */
sfxge_ev_qcomplete(evq, B_TRUE);
/* Re-prime the event queue for interrupts */
if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
goto fail;
SFXGE_EVQ_UNLOCK(evq);
return (0);
fail:
SFXGE_EVQ_UNLOCK(evq);
return (rc);
}
static boolean_t
sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
uint16_t flags)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
struct sfxge_rxq *rxq;
unsigned int stop;
unsigned int delta;
struct sfxge_rx_sw_desc *rx_desc;
evq = arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
if (evq->exception)
goto done;
rxq = sc->rxq[label];
KASSERT(rxq != NULL, ("rxq == NULL"));
KASSERT(evq->index == rxq->index,
("evq->index != rxq->index"));
if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
goto done;
stop = (id + 1) & rxq->ptr_mask;
id = rxq->pending & rxq->ptr_mask;
delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
rxq->pending += delta;
if (delta != 1) {
if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) ||
(delta <= 0) ||
(delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
evq->exception = B_TRUE;
device_printf(sc->dev, "RX completion out of order"
" (id=%#x delta=%u flags=%#x); resetting\n",
id, delta, flags);
sfxge_schedule_reset(sc);
goto done;
}
}
rx_desc = &rxq->queue[id];
prefetch_read_many(rx_desc->mbuf);
for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
rx_desc = &rxq->queue[id];
KASSERT(rx_desc->flags == EFX_DISCARD,
("rx_desc->flags != EFX_DISCARD"));
rx_desc->flags = flags;
KASSERT(size < (1 << 16), ("size > (1 << 16)"));
rx_desc->size = (uint16_t)size;
}
evq->rx_done++;
if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
sfxge_ev_qcomplete(evq, B_FALSE);
done:
return (evq->rx_done >= SFXGE_EV_BATCH);
}
