void sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n, uint32_t *idp) { KASSERT(sc->buffer_table_next + n <= efx_nic_cfg_get(sc->enp)->enc_buftbl_limit, ("buffer table full")); *idp = sc->buffer_table_next; sc->buffer_table_next += n; }
static int sfxge_ifnet_init(struct ifnet *ifp, struct sfxge_softc *sc) { const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp); device_t dev; int rc; dev = sc->dev; sc->ifnet = ifp; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_init = sfxge_if_init; ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = sfxge_if_ioctl; ifp->if_capabilities = SFXGE_CAP; ifp->if_capenable = SFXGE_CAP_ENABLE; ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; ether_ifattach(ifp, encp->enc_mac_addr); #ifdef SFXGE_HAVE_MQ ifp->if_transmit = sfxge_if_transmit; ifp->if_qflush = sfxge_if_qflush; #else ifp->if_start = sfxge_if_start; IFQ_SET_MAXLEN(&ifp->if_snd, SFXGE_NDESCS - 1); ifp->if_snd.ifq_drv_maxlen = SFXGE_NDESCS - 1; IFQ_SET_READY(&ifp->if_snd); mtx_init(&sc->tx_lock, "txq", NULL, MTX_DEF); #endif if ((rc = sfxge_port_ifmedia_init(sc)) != 0) goto fail; return 0; fail: ether_ifdetach(sc->ifnet); return rc; }
static int sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; struct sfxge_intr *intr = &sc->intr; unsigned int moderation; int error; int index; sx_xlock(&sc->softc_lock); if (req->newptr) { if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) != 0) goto out; /* We may not be calling efx_ev_qmoderate() now, * so we have to range-check the value ourselves. */ if (moderation > efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { error = EINVAL; goto out; } sc->ev_moderation = moderation; if (intr->state == SFXGE_INTR_STARTED) { for (index = 0; index < intr->n_alloc; index++) sfxge_ev_qmoderate(sc, index, moderation); } } else { error = SYSCTL_OUT(req, &sc->ev_moderation, sizeof(sc->ev_moderation)); } out: sx_xunlock(&sc->softc_lock); return error; }
static int sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; struct sfxge_intr *intr = &sc->intr; unsigned int moderation; int error; unsigned int index; SFXGE_ADAPTER_LOCK(sc); if (req->newptr != NULL) { if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) != 0) goto out; /* We may not be calling efx_ev_qmoderate() now, * so we have to range-check the value ourselves. */ if (moderation > efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { error = EINVAL; goto out; } sc->ev_moderation = moderation; if (intr->state == SFXGE_INTR_STARTED) { for (index = 0; index < sc->evq_count; index++) sfxge_ev_qmoderate(sc, index, moderation); } } else { error = SYSCTL_OUT(req, &sc->ev_moderation, sizeof(sc->ev_moderation)); } out: SFXGE_ADAPTER_UNLOCK(sc); return (error); }
int sfc_tx_start(struct sfc_adapter *sa) { unsigned int sw_index; int rc = 0; sfc_log_init(sa, "txq_count = %u", sa->txq_count); if (sa->tso) { if (!efx_nic_cfg_get(sa->nic)->enc_fw_assisted_tso_v2_enabled) { sfc_warn(sa, "TSO support was unable to be restored"); sa->tso = B_FALSE; } } rc = efx_tx_init(sa->nic); if (rc != 0) goto fail_efx_tx_init; for (sw_index = 0; sw_index < sa->txq_count; ++sw_index) { if (!(sa->txq_info[sw_index].deferred_start) || sa->txq_info[sw_index].deferred_started) { rc = sfc_tx_qstart(sa, sw_index); if (rc != 0) goto fail_tx_qstart; } } return 0; fail_tx_qstart: while (sw_index-- > 0) sfc_tx_qstop(sa, sw_index); efx_tx_fini(sa->nic); fail_efx_tx_init: sfc_log_init(sa, "failed (rc = %d)", rc); return rc; }
/* * The function is used to insert or update VLAN tag; * the firmware has state of the firmware tag to insert per TxQ * (controlled by option descriptors), hence, if the tag of the * packet to be sent is different from one remembered by the firmware, * the function will update it */ static unsigned int sfc_efx_tx_maybe_insert_tag(struct sfc_efx_txq *txq, struct rte_mbuf *m, efx_desc_t **pend) { uint16_t this_tag = ((m->ol_flags & PKT_TX_VLAN_PKT) ? m->vlan_tci : 0); if (this_tag == txq->hw_vlan_tci) return 0; /* * The expression inside SFC_ASSERT() is not desired to be checked in * a non-debug build because it might be too expensive on the data path */ SFC_ASSERT(efx_nic_cfg_get(txq->evq->sa->nic)->enc_hw_tx_insert_vlan_enabled); efx_tx_qdesc_vlantci_create(txq->common, rte_cpu_to_be_16(this_tag), *pend); (*pend)++; txq->hw_vlan_tci = this_tag; return 1; }
static boolean_t sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, uint16_t flags) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int stop; unsigned int delta; struct sfxge_rx_sw_desc *rx_desc; evq = arg; SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); sc = evq->sc; if (evq->exception) goto done; rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) goto done; stop = (id + 1) & rxq->ptr_mask; id = rxq->pending & rxq->ptr_mask; delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); rxq->pending += delta; if (delta != 1) { if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) || (delta <= 0) || (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { evq->exception = B_TRUE; device_printf(sc->dev, "RX completion out of order" " (id=%#x delta=%u flags=%#x); resetting\n", id, delta, flags); sfxge_schedule_reset(sc); goto done; } } rx_desc = &rxq->queue[id]; prefetch_read_many(rx_desc->mbuf); for (; id != stop; id = (id + 1) & rxq->ptr_mask) { rx_desc = &rxq->queue[id]; KASSERT(rx_desc->flags == EFX_DISCARD, ("rx_desc->flags != EFX_DISCARD")); rx_desc->flags = flags; KASSERT(size < (1 << 16), ("size > (1 << 16)")); rx_desc->size = (uint16_t)size; } evq->rx_done++; if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) sfxge_ev_qcomplete(evq, B_FALSE); done: return (evq->rx_done >= SFXGE_EV_BATCH); }
static int sfc_tx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_tx_desc, const struct rte_eth_txconf *tx_conf) { unsigned int flags = tx_conf->txq_flags; const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); int rc = 0; if (tx_conf->tx_rs_thresh != 0) { sfc_err(sa, "RS bit in transmit descriptor is not supported"); rc = EINVAL; } if (tx_conf->tx_free_thresh > EFX_TXQ_LIMIT(nb_tx_desc)) { sfc_err(sa, "TxQ free threshold too large: %u vs maximum %u", tx_conf->tx_free_thresh, EFX_TXQ_LIMIT(nb_tx_desc)); rc = EINVAL; } if (tx_conf->tx_thresh.pthresh != 0 || tx_conf->tx_thresh.hthresh != 0 || tx_conf->tx_thresh.wthresh != 0) { sfc_err(sa, "prefetch/host/writeback thresholds are not supported"); rc = EINVAL; } if (((flags & ETH_TXQ_FLAGS_NOMULTSEGS) == 0) && (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_SEG)) { sfc_err(sa, "Multi-segment is not supported by %s datapath", sa->dp_tx->dp.name); rc = EINVAL; } if ((flags & ETH_TXQ_FLAGS_NOVLANOFFL) == 0) { if (!encp->enc_hw_tx_insert_vlan_enabled) { sfc_err(sa, "VLAN offload is not supported"); rc = EINVAL; } else if (~sa->dp_tx->features & SFC_DP_TX_FEAT_VLAN_INSERT) { sfc_err(sa, "VLAN offload is not supported by %s datapath", sa->dp_tx->dp.name); rc = EINVAL; } } if ((flags & ETH_TXQ_FLAGS_NOXSUMSCTP) == 0) { sfc_err(sa, "SCTP offload is not supported"); rc = EINVAL; } /* We either perform both TCP and UDP offload, or no offload at all */ if (((flags & ETH_TXQ_FLAGS_NOXSUMTCP) == 0) != ((flags & ETH_TXQ_FLAGS_NOXSUMUDP) == 0)) { sfc_err(sa, "TCP and UDP offloads can't be set independently"); rc = EINVAL; } return rc; }
int sfc_tx_configure(struct sfc_adapter *sa) { const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); const struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf; const unsigned int nb_tx_queues = sa->eth_dev->data->nb_tx_queues; int rc = 0; sfc_log_init(sa, "nb_tx_queues=%u (old %u)", nb_tx_queues, sa->txq_count); /* * The datapath implementation assumes absence of boundary * limits on Tx DMA descriptors. Addition of these checks on * datapath would simply make the datapath slower. */ if (encp->enc_tx_dma_desc_boundary != 0) { rc = ENOTSUP; goto fail_tx_dma_desc_boundary; } rc = sfc_tx_check_mode(sa, &dev_conf->txmode); if (rc != 0) goto fail_check_mode; if (nb_tx_queues == sa->txq_count) goto done; if (sa->txq_info == NULL) { sa->txq_info = rte_calloc_socket("sfc-txqs", nb_tx_queues, sizeof(sa->txq_info[0]), 0, sa->socket_id); if (sa->txq_info == NULL) goto fail_txqs_alloc; } else { struct sfc_txq_info *new_txq_info; if (nb_tx_queues < sa->txq_count) sfc_tx_fini_queues(sa, nb_tx_queues); new_txq_info = rte_realloc(sa->txq_info, nb_tx_queues * sizeof(sa->txq_info[0]), 0); if (new_txq_info == NULL && nb_tx_queues > 0) goto fail_txqs_realloc; sa->txq_info = new_txq_info; if (nb_tx_queues > sa->txq_count) memset(&sa->txq_info[sa->txq_count], 0, (nb_tx_queues - sa->txq_count) * sizeof(sa->txq_info[0])); } while (sa->txq_count < nb_tx_queues) { rc = sfc_tx_qinit_info(sa, sa->txq_count); if (rc != 0) goto fail_tx_qinit_info; sa->txq_count++; } done: return 0; fail_tx_qinit_info: fail_txqs_realloc: fail_txqs_alloc: sfc_tx_close(sa); fail_check_mode: fail_tx_dma_desc_boundary: sfc_log_init(sa, "failed (rc = %d)", rc); return rc; }
int sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index, uint16_t nb_tx_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf) { const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); struct sfc_txq_info *txq_info; struct sfc_evq *evq; struct sfc_txq *txq; int rc = 0; struct sfc_dp_tx_qcreate_info info; sfc_log_init(sa, "TxQ = %u", sw_index); rc = sfc_tx_qcheck_conf(sa, nb_tx_desc, tx_conf); if (rc != 0) goto fail_bad_conf; SFC_ASSERT(sw_index < sa->txq_count); txq_info = &sa->txq_info[sw_index]; SFC_ASSERT(nb_tx_desc <= sa->txq_max_entries); txq_info->entries = nb_tx_desc; rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_TX, sw_index, txq_info->entries, socket_id, &evq); if (rc != 0) goto fail_ev_qinit; rc = ENOMEM; txq = rte_zmalloc_socket("sfc-txq", sizeof(*txq), 0, socket_id); if (txq == NULL) goto fail_txq_alloc; txq_info->txq = txq; txq->hw_index = sw_index; txq->evq = evq; txq->free_thresh = (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh : SFC_TX_DEFAULT_FREE_THRESH; txq->flags = tx_conf->txq_flags; rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries), socket_id, &txq->mem); if (rc != 0) goto fail_dma_alloc; memset(&info, 0, sizeof(info)); info.free_thresh = txq->free_thresh; info.flags = tx_conf->txq_flags; info.txq_entries = txq_info->entries; info.dma_desc_size_max = encp->enc_tx_dma_desc_size_max; info.txq_hw_ring = txq->mem.esm_base; info.evq_entries = txq_info->entries; info.evq_hw_ring = evq->mem.esm_base; info.hw_index = txq->hw_index; info.mem_bar = sa->mem_bar.esb_base; rc = sa->dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index, &SFC_DEV_TO_PCI(sa->eth_dev)->addr, socket_id, &info, &txq->dp); if (rc != 0) goto fail_dp_tx_qinit; evq->dp_txq = txq->dp; txq->state = SFC_TXQ_INITIALIZED; txq_info->deferred_start = (tx_conf->tx_deferred_start != 0); return 0; fail_dp_tx_qinit: sfc_dma_free(sa, &txq->mem); fail_dma_alloc: txq_info->txq = NULL; rte_free(txq); fail_txq_alloc: sfc_ev_qfini(evq); fail_ev_qinit: txq_info->entries = 0; fail_bad_conf: sfc_log_init(sa, "failed (TxQ = %u, rc = %d)", sw_index, rc); return rc; }
int sfxge_mcdi_ioctl(struct sfxge_softc *sc, sfxge_ioc_t *ip) { const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp); struct sfxge_mcdi *mp = &(sc->mcdi); efx_mcdi_req_t emr; uint8_t *mcdibuf; int rc; if (mp->state == SFXGE_MCDI_UNINITIALIZED) { rc = ENODEV; goto fail1; } if (!(encp->enc_features & EFX_FEATURE_MCDI)) { rc = ENOTSUP; goto fail2; } if (ip->u.mcdi.len > SFXGE_MCDI_MAX_PAYLOAD) { rc = EINVAL; goto fail3; } mcdibuf = malloc(SFXGE_MCDI_MAX_PAYLOAD, M_TEMP, M_WAITOK | M_ZERO); if ((rc = copyin(ip->u.mcdi.payload, mcdibuf, ip->u.mcdi.len)) != 0) { goto fail5; } emr.emr_cmd = ip->u.mcdi.cmd; emr.emr_in_buf = mcdibuf; emr.emr_in_length = ip->u.mcdi.len; emr.emr_out_buf = mcdibuf; emr.emr_out_length = SFXGE_MCDI_MAX_PAYLOAD; sfxge_mcdi_execute(sc, &emr); ip->u.mcdi.rc = emr.emr_rc; ip->u.mcdi.cmd = emr.emr_cmd; ip->u.mcdi.len = emr.emr_out_length_used; if ((rc = copyout(mcdibuf, ip->u.mcdi.payload, ip->u.mcdi.len)) != 0) { goto fail6; } /* * Helpfully trigger a device reset in response to an MCDI_CMD_REBOOT * Both ports will see ->emt_exception callbacks on the next MCDI poll */ if (ip->u.mcdi.cmd == MC_CMD_REBOOT) { EFSYS_PROBE(mcdi_ioctl_mc_reboot); /* sfxge_t->s_state_lock held */ (void) sfxge_schedule_reset(sc); } free(mcdibuf, M_TEMP); return (0); fail6: fail5: free(mcdibuf, M_TEMP); fail3: fail2: fail1: return (rc); }
int sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx, struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs, size_t *pkt_len) { uint8_t *tsoh; const struct tcp_hdr *th; efsys_dma_addr_t header_paddr; uint16_t packet_id; uint32_t sent_seq; struct rte_mbuf *m = *in_seg; size_t nh_off = m->l2_len; /* IP header offset */ size_t tcph_off = m->l2_len + m->l3_len; /* TCP header offset */ size_t header_len = m->l2_len + m->l3_len + m->l4_len; const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->evq->sa->nic); idx += SFC_TSO_OPT_DESCS_NUM; /* Packets which have too big headers should be discarded */ if (unlikely(header_len > SFC_TSOH_STD_LEN)) return EMSGSIZE; /* * The TCP header must start at most 208 bytes into the frame. * If it starts later than this then the NIC won't realise * it's a TCP packet and TSO edits won't be applied */ if (unlikely(tcph_off > encp->enc_tx_tso_tcp_header_offset_limit)) return EMSGSIZE; header_paddr = rte_pktmbuf_iova(m); /* * Sometimes headers may be split across multiple mbufs. In such cases * we need to glue those pieces and store them in some temporary place. * Also, packet headers must be contiguous in memory, so that * they can be referred to with a single DMA descriptor. EF10 has no * limitations on address boundaries crossing by DMA descriptor data. */ if (m->data_len < header_len) { tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh; sfc_tso_prepare_header(tsoh, header_len, in_seg, in_off); header_paddr = rte_malloc_virt2iova((void *)tsoh); } else { if (m->data_len == header_len) { *in_off = 0; *in_seg = m->next; } else { *in_off = header_len; } tsoh = rte_pktmbuf_mtod(m, uint8_t *); } /* Handle IP header */ if (m->ol_flags & PKT_TX_IPV4) { const struct ipv4_hdr *iphe4; iphe4 = (const struct ipv4_hdr *)(tsoh + nh_off); rte_memcpy(&packet_id, &iphe4->packet_id, sizeof(uint16_t)); packet_id = rte_be_to_cpu_16(packet_id); } else if (m->ol_flags & PKT_TX_IPV6) { packet_id = 0; } else { return EINVAL; } /* Handle TCP header */ th = (const struct tcp_hdr *)(tsoh + tcph_off); rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t)); sent_seq = rte_be_to_cpu_32(sent_seq); efx_tx_qdesc_tso2_create(txq->common, packet_id, 0, sent_seq, m->tso_segsz, *pend, EFX_TX_FATSOV2_OPT_NDESCS); *pend += EFX_TX_FATSOV2_OPT_NDESCS; *pkt_descs += EFX_TX_FATSOV2_OPT_NDESCS; efx_tx_qdesc_dma_create(txq->common, header_paddr, header_len, B_FALSE, (*pend)++); (*pkt_descs)++; *pkt_len -= header_len; return 0; }