static boolean_t
sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
struct sfxge_rxq *rxq;
unsigned int index;
unsigned int label;
uint16_t magic;
evq = (struct sfxge_evq *)arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
rxq = sc->rxq[rxq_index];
KASSERT(rxq != NULL, ("rxq == NULL"));
/* Resend a software event on the correct queue */
index = rxq->index;
evq = sc->evq[index];
label = rxq_index;
KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
("evq not started"));
efx_ev_qpost(evq->common, magic);
return (B_FALSE);
}
static boolean_t
sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
evq = (struct sfxge_evq *)arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
DBGPRINT(sc->dev, "[%d] %s", evq->index,
(code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
(code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
(code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
(code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
(code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
(code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
(code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
(code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
(code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
"UNKNOWN");
evq->exception = B_TRUE;
if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
device_printf(sc->dev,
"hardware exception (code=%u); resetting\n",
code);
sfxge_schedule_reset(sc);
}
return (B_FALSE);
}
static boolean_t
sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
evq = (struct sfxge_evq *)arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
sfxge_mac_link_update(sc, link_mode);
return (0);
}
static boolean_t
sfxge_ev_initialized(void *arg)
{
struct sfxge_evq *evq;
evq = (struct sfxge_evq *)arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
/* Init done events may be duplicated on 7xxx */
KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
evq->init_state == SFXGE_EVQ_STARTED,
("evq not starting"));
evq->init_state = SFXGE_EVQ_STARTED;
return (0);
}
static boolean_t
sfxge_ev_software(void *arg, uint16_t magic)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
unsigned int label;
evq = (struct sfxge_evq *)arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
switch (magic) {
case SFXGE_MAGIC_RX_QFLUSH_DONE:
sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
break;
case SFXGE_MAGIC_RX_QFLUSH_FAILED:
sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
break;
case SFXGE_MAGIC_RX_QREFILL:
sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
break;
case SFXGE_MAGIC_TX_QFLUSH_DONE: {
struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
KASSERT(txq != NULL, ("txq == NULL"));
KASSERT(evq->index == txq->evq_index,
("evq->index != txq->evq_index"));
sfxge_tx_qflush_done(txq);
break;
}
default:
break;
}
return (B_FALSE);
}
static boolean_t
sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
{
struct sfxge_evq *evq;
struct sfxge_txq *txq;
unsigned int stop;
unsigned int delta;
evq = (struct sfxge_evq *)arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
txq = sfxge_get_txq_by_label(evq, label);
KASSERT(txq != NULL, ("txq == NULL"));
KASSERT(evq->index == txq->evq_index,
("evq->index != txq->evq_index"));
if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
goto done;
stop = (id + 1) & txq->ptr_mask;
id = txq->pending & txq->ptr_mask;
delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
txq->pending += delta;
evq->tx_done++;
if (txq->next == NULL &&
evq->txqs != &(txq->next)) {
*(evq->txqs) = txq;
evq->txqs = &(txq->next);
}
if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
sfxge_tx_qcomplete(txq, evq);
done:
return (evq->tx_done >= SFXGE_EV_BATCH);
}
static boolean_t
sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
struct sfxge_txq *txq;
unsigned int label;
uint16_t magic;
evq = (struct sfxge_evq *)arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
txq = sc->txq[txq_index];
KASSERT(txq != NULL, ("txq == NULL"));
KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
("txq not initialized"));
if (txq->evq_index == evq->index) {
sfxge_tx_qflush_done(txq);
return (B_FALSE);
}
/* Resend a software event on the correct queue */
evq = sc->evq[txq->evq_index];
label = txq->type;
KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label,
("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label"));
magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
("evq not started"));
efx_ev_qpost(evq->common, magic);
return (B_FALSE);
}
static void
sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
{
struct sfxge_softc *sc;
unsigned int index;
struct sfxge_rxq *rxq;
struct sfxge_txq *txq;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
index = evq->index;
rxq = sc->rxq[index];
if ((txq = evq->txq) != NULL) {
evq->txq = NULL;
evq->txqs = &(evq->txq);
do {
struct sfxge_txq *next;
next = txq->next;
txq->next = NULL;
KASSERT(txq->evq_index == index,
("txq->evq_index != index"));
if (txq->pending != txq->completed)
sfxge_tx_qcomplete(txq, evq);
txq = next;
} while (txq != NULL);
}
if (rxq->pending != rxq->completed)
sfxge_rx_qcomplete(rxq, eop);
}
static boolean_t
sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
uint16_t flags)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
struct sfxge_rxq *rxq;
unsigned int stop;
unsigned int delta;
struct sfxge_rx_sw_desc *rx_desc;
evq = arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
if (evq->exception)
goto done;
rxq = sc->rxq[label];
KASSERT(rxq != NULL, ("rxq == NULL"));
KASSERT(evq->index == rxq->index,
("evq->index != rxq->index"));
if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
goto done;
stop = (id + 1) & rxq->ptr_mask;
id = rxq->pending & rxq->ptr_mask;
delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
rxq->pending += delta;
if (delta != 1) {
if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) ||
(delta <= 0) ||
(delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
evq->exception = B_TRUE;
device_printf(sc->dev, "RX completion out of order"
" (id=%#x delta=%u flags=%#x); resetting\n",
id, delta, flags);
sfxge_schedule_reset(sc);
goto done;
}
}
rx_desc = &rxq->queue[id];
prefetch_read_many(rx_desc->mbuf);
for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
rx_desc = &rxq->queue[id];
KASSERT(rx_desc->flags == EFX_DISCARD,
("rx_desc->flags != EFX_DISCARD"));
rx_desc->flags = flags;
KASSERT(size < (1 << 16), ("size > (1 << 16)"));
rx_desc->size = (uint16_t)size;
}
evq->rx_done++;
if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
sfxge_ev_qcomplete(evq, B_FALSE);
done:
return (evq->rx_done >= SFXGE_EV_BATCH);
}
static void
sfxge_rx_qfill(struct sfxge_rxq *rxq, unsigned int target, boolean_t retrying)
{
struct sfxge_softc *sc;
unsigned int index;
struct sfxge_evq *evq;
unsigned int batch;
unsigned int rxfill;
unsigned int mblksize;
int ntodo;
efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
sc = rxq->sc;
index = rxq->index;
evq = sc->evq[index];
prefetch_read_many(sc->enp);
prefetch_read_many(rxq->common);
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
return;
rxfill = rxq->added - rxq->completed;
KASSERT(rxfill <= EFX_RXQ_LIMIT(rxq->entries),
("rxfill > EFX_RXQ_LIMIT(rxq->entries)"));
ntodo = min(EFX_RXQ_LIMIT(rxq->entries) - rxfill, target);
KASSERT(ntodo <= EFX_RXQ_LIMIT(rxq->entries),
("ntodo > EFX_RQX_LIMIT(rxq->entries)"));
if (ntodo == 0)
return;
batch = 0;
mblksize = sc->rx_buffer_size - sc->rx_buffer_align;
while (ntodo-- > 0) {
unsigned int id;
struct sfxge_rx_sw_desc *rx_desc;
bus_dma_segment_t seg;
struct mbuf *m;
id = (rxq->added + batch) & rxq->ptr_mask;
rx_desc = &rxq->queue[id];
KASSERT(rx_desc->mbuf == NULL, ("rx_desc->mbuf != NULL"));
rx_desc->flags = EFX_DISCARD;
m = rx_desc->mbuf = sfxge_rx_alloc_mbuf(sc);
if (m == NULL)
break;
/* m_len specifies length of area to be mapped for DMA */
m->m_len = mblksize;
m->m_data = (caddr_t)P2ROUNDUP((uintptr_t)m->m_data, CACHE_LINE_SIZE);
m->m_data += sc->rx_buffer_align;
sfxge_map_mbuf_fast(rxq->mem.esm_tag, rxq->mem.esm_map, m, &seg);
addr[batch++] = seg.ds_addr;
if (batch == SFXGE_REFILL_BATCH) {
efx_rx_qpost(rxq->common, addr, mblksize, batch,
rxq->completed, rxq->added);
rxq->added += batch;
batch = 0;
}
}
if (ntodo != 0)
sfxge_rx_schedule_refill(rxq, retrying);
if (batch != 0) {
efx_rx_qpost(rxq->common, addr, mblksize, batch,
rxq->completed, rxq->added);
rxq->added += batch;
}
/* Make the descriptors visible to the hardware */
bus_dmamap_sync(rxq->mem.esm_tag, rxq->mem.esm_map,
BUS_DMASYNC_PREWRITE);
efx_rx_qpush(rxq->common, rxq->added, &rxq->pushed);
/* The queue could still be empty if no descriptors were actually
* pushed, in which case there will be no event to cause the next
* refill, so we must schedule a refill ourselves.
*/
if(rxq->pushed == rxq->completed) {
sfxge_rx_schedule_refill(rxq, retrying);
}
}