void
sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n, uint32_t *idp)
{
KASSERT(sc->buffer_table_next + n <=
efx_nic_cfg_get(sc->enp)->enc_buftbl_limit,
("buffer table full"));
*idp = sc->buffer_table_next;
sc->buffer_table_next += n;
}
static int
sfxge_ifnet_init(struct ifnet *ifp, struct sfxge_softc *sc)
{
const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp);
device_t dev;
int rc;
dev = sc->dev;
sc->ifnet = ifp;
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_init = sfxge_if_init;
ifp->if_softc = sc;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = sfxge_if_ioctl;
ifp->if_capabilities = SFXGE_CAP;
ifp->if_capenable = SFXGE_CAP_ENABLE;
ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO;
ether_ifattach(ifp, encp->enc_mac_addr);
#ifdef SFXGE_HAVE_MQ
ifp->if_transmit = sfxge_if_transmit;
ifp->if_qflush = sfxge_if_qflush;
#else
ifp->if_start = sfxge_if_start;
IFQ_SET_MAXLEN(&ifp->if_snd, SFXGE_NDESCS - 1);
ifp->if_snd.ifq_drv_maxlen = SFXGE_NDESCS - 1;
IFQ_SET_READY(&ifp->if_snd);
mtx_init(&sc->tx_lock, "txq", NULL, MTX_DEF);
#endif
if ((rc = sfxge_port_ifmedia_init(sc)) != 0)
goto fail;
return 0;
fail:
ether_ifdetach(sc->ifnet);
return rc;
}
static int
sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
{
struct sfxge_softc *sc = arg1;
struct sfxge_intr *intr = &sc->intr;
unsigned int moderation;
int error;
int index;
sx_xlock(&sc->softc_lock);
if (req->newptr) {
if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
!= 0)
goto out;
/* We may not be calling efx_ev_qmoderate() now,
* so we have to range-check the value ourselves.
*/
if (moderation >
efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) {
error = EINVAL;
goto out;
}
sc->ev_moderation = moderation;
if (intr->state == SFXGE_INTR_STARTED) {
for (index = 0; index < intr->n_alloc; index++)
sfxge_ev_qmoderate(sc, index, moderation);
}
} else {
error = SYSCTL_OUT(req, &sc->ev_moderation,
sizeof(sc->ev_moderation));
}
out:
sx_xunlock(&sc->softc_lock);
return error;
}
static int
sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
{
struct sfxge_softc *sc = arg1;
struct sfxge_intr *intr = &sc->intr;
unsigned int moderation;
int error;
unsigned int index;
SFXGE_ADAPTER_LOCK(sc);
if (req->newptr != NULL) {
if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
!= 0)
goto out;
/* We may not be calling efx_ev_qmoderate() now,
* so we have to range-check the value ourselves.
*/
if (moderation >
efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
error = EINVAL;
goto out;
}
sc->ev_moderation = moderation;
if (intr->state == SFXGE_INTR_STARTED) {
for (index = 0; index < sc->evq_count; index++)
sfxge_ev_qmoderate(sc, index, moderation);
}
} else {
error = SYSCTL_OUT(req, &sc->ev_moderation,
sizeof(sc->ev_moderation));
}
out:
SFXGE_ADAPTER_UNLOCK(sc);
return (error);
}
int
sfc_tx_start(struct sfc_adapter *sa)
{
unsigned int sw_index;
int rc = 0;
sfc_log_init(sa, "txq_count = %u", sa->txq_count);
if (sa->tso) {
if (!efx_nic_cfg_get(sa->nic)->enc_fw_assisted_tso_v2_enabled) {
sfc_warn(sa, "TSO support was unable to be restored");
sa->tso = B_FALSE;
}
}
rc = efx_tx_init(sa->nic);
if (rc != 0)
goto fail_efx_tx_init;
for (sw_index = 0; sw_index < sa->txq_count; ++sw_index) {
if (!(sa->txq_info[sw_index].deferred_start) ||
sa->txq_info[sw_index].deferred_started) {
rc = sfc_tx_qstart(sa, sw_index);
if (rc != 0)
goto fail_tx_qstart;
}
}
return 0;
fail_tx_qstart:
while (sw_index-- > 0)
sfc_tx_qstop(sa, sw_index);
efx_tx_fini(sa->nic);
fail_efx_tx_init:
sfc_log_init(sa, "failed (rc = %d)", rc);
return rc;
}
/*
* The function is used to insert or update VLAN tag;
* the firmware has state of the firmware tag to insert per TxQ
* (controlled by option descriptors), hence, if the tag of the
* packet to be sent is different from one remembered by the firmware,
* the function will update it
*/
static unsigned int
sfc_efx_tx_maybe_insert_tag(struct sfc_efx_txq *txq, struct rte_mbuf *m,
efx_desc_t **pend)
{
uint16_t this_tag = ((m->ol_flags & PKT_TX_VLAN_PKT) ?
m->vlan_tci : 0);
if (this_tag == txq->hw_vlan_tci)
return 0;
/*
* The expression inside SFC_ASSERT() is not desired to be checked in
* a non-debug build because it might be too expensive on the data path
*/
SFC_ASSERT(efx_nic_cfg_get(txq->evq->sa->nic)->enc_hw_tx_insert_vlan_enabled);
efx_tx_qdesc_vlantci_create(txq->common, rte_cpu_to_be_16(this_tag),
*pend);
(*pend)++;
txq->hw_vlan_tci = this_tag;
return 1;
}
static boolean_t
sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
uint16_t flags)
{
struct sfxge_evq *evq;
struct sfxge_softc *sc;
struct sfxge_rxq *rxq;
unsigned int stop;
unsigned int delta;
struct sfxge_rx_sw_desc *rx_desc;
evq = arg;
SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
sc = evq->sc;
if (evq->exception)
goto done;
rxq = sc->rxq[label];
KASSERT(rxq != NULL, ("rxq == NULL"));
KASSERT(evq->index == rxq->index,
("evq->index != rxq->index"));
if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
goto done;
stop = (id + 1) & rxq->ptr_mask;
id = rxq->pending & rxq->ptr_mask;
delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
rxq->pending += delta;
if (delta != 1) {
if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) ||
(delta <= 0) ||
(delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
evq->exception = B_TRUE;
device_printf(sc->dev, "RX completion out of order"
" (id=%#x delta=%u flags=%#x); resetting\n",
id, delta, flags);
sfxge_schedule_reset(sc);
goto done;
}
}
rx_desc = &rxq->queue[id];
prefetch_read_many(rx_desc->mbuf);
for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
rx_desc = &rxq->queue[id];
KASSERT(rx_desc->flags == EFX_DISCARD,
("rx_desc->flags != EFX_DISCARD"));
rx_desc->flags = flags;
KASSERT(size < (1 << 16), ("size > (1 << 16)"));
rx_desc->size = (uint16_t)size;
}
evq->rx_done++;
if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
sfxge_ev_qcomplete(evq, B_FALSE);
done:
return (evq->rx_done >= SFXGE_EV_BATCH);
}
static int
sfc_tx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_tx_desc,
const struct rte_eth_txconf *tx_conf)
{
unsigned int flags = tx_conf->txq_flags;
const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
int rc = 0;
if (tx_conf->tx_rs_thresh != 0) {
sfc_err(sa, "RS bit in transmit descriptor is not supported");
rc = EINVAL;
}
if (tx_conf->tx_free_thresh > EFX_TXQ_LIMIT(nb_tx_desc)) {
sfc_err(sa,
"TxQ free threshold too large: %u vs maximum %u",
tx_conf->tx_free_thresh, EFX_TXQ_LIMIT(nb_tx_desc));
rc = EINVAL;
}
if (tx_conf->tx_thresh.pthresh != 0 ||
tx_conf->tx_thresh.hthresh != 0 ||
tx_conf->tx_thresh.wthresh != 0) {
sfc_err(sa,
"prefetch/host/writeback thresholds are not supported");
rc = EINVAL;
}
if (((flags & ETH_TXQ_FLAGS_NOMULTSEGS) == 0) &&
(~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_SEG)) {
sfc_err(sa, "Multi-segment is not supported by %s datapath",
sa->dp_tx->dp.name);
rc = EINVAL;
}
if ((flags & ETH_TXQ_FLAGS_NOVLANOFFL) == 0) {
if (!encp->enc_hw_tx_insert_vlan_enabled) {
sfc_err(sa, "VLAN offload is not supported");
rc = EINVAL;
} else if (~sa->dp_tx->features & SFC_DP_TX_FEAT_VLAN_INSERT) {
sfc_err(sa,
"VLAN offload is not supported by %s datapath",
sa->dp_tx->dp.name);
rc = EINVAL;
}
}
if ((flags & ETH_TXQ_FLAGS_NOXSUMSCTP) == 0) {
sfc_err(sa, "SCTP offload is not supported");
rc = EINVAL;
}
/* We either perform both TCP and UDP offload, or no offload at all */
if (((flags & ETH_TXQ_FLAGS_NOXSUMTCP) == 0) !=
((flags & ETH_TXQ_FLAGS_NOXSUMUDP) == 0)) {
sfc_err(sa, "TCP and UDP offloads can't be set independently");
rc = EINVAL;
}
return rc;
}
int
sfc_tx_configure(struct sfc_adapter *sa)
{
const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
const struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf;
const unsigned int nb_tx_queues = sa->eth_dev->data->nb_tx_queues;
int rc = 0;
sfc_log_init(sa, "nb_tx_queues=%u (old %u)",
nb_tx_queues, sa->txq_count);
/*
* The datapath implementation assumes absence of boundary
* limits on Tx DMA descriptors. Addition of these checks on
* datapath would simply make the datapath slower.
*/
if (encp->enc_tx_dma_desc_boundary != 0) {
rc = ENOTSUP;
goto fail_tx_dma_desc_boundary;
}
rc = sfc_tx_check_mode(sa, &dev_conf->txmode);
if (rc != 0)
goto fail_check_mode;
if (nb_tx_queues == sa->txq_count)
goto done;
if (sa->txq_info == NULL) {
sa->txq_info = rte_calloc_socket("sfc-txqs", nb_tx_queues,
sizeof(sa->txq_info[0]), 0,
sa->socket_id);
if (sa->txq_info == NULL)
goto fail_txqs_alloc;
} else {
struct sfc_txq_info *new_txq_info;
if (nb_tx_queues < sa->txq_count)
sfc_tx_fini_queues(sa, nb_tx_queues);
new_txq_info =
rte_realloc(sa->txq_info,
nb_tx_queues * sizeof(sa->txq_info[0]), 0);
if (new_txq_info == NULL && nb_tx_queues > 0)
goto fail_txqs_realloc;
sa->txq_info = new_txq_info;
if (nb_tx_queues > sa->txq_count)
memset(&sa->txq_info[sa->txq_count], 0,
(nb_tx_queues - sa->txq_count) *
sizeof(sa->txq_info[0]));
}
while (sa->txq_count < nb_tx_queues) {
rc = sfc_tx_qinit_info(sa, sa->txq_count);
if (rc != 0)
goto fail_tx_qinit_info;
sa->txq_count++;
}
done:
return 0;
fail_tx_qinit_info:
fail_txqs_realloc:
fail_txqs_alloc:
sfc_tx_close(sa);
fail_check_mode:
fail_tx_dma_desc_boundary:
sfc_log_init(sa, "failed (rc = %d)", rc);
return rc;
}
int
sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
uint16_t nb_tx_desc, unsigned int socket_id,
const struct rte_eth_txconf *tx_conf)
{
const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
struct sfc_txq_info *txq_info;
struct sfc_evq *evq;
struct sfc_txq *txq;
int rc = 0;
struct sfc_dp_tx_qcreate_info info;
sfc_log_init(sa, "TxQ = %u", sw_index);
rc = sfc_tx_qcheck_conf(sa, nb_tx_desc, tx_conf);
if (rc != 0)
goto fail_bad_conf;
SFC_ASSERT(sw_index < sa->txq_count);
txq_info = &sa->txq_info[sw_index];
SFC_ASSERT(nb_tx_desc <= sa->txq_max_entries);
txq_info->entries = nb_tx_desc;
rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_TX, sw_index,
txq_info->entries, socket_id, &evq);
if (rc != 0)
goto fail_ev_qinit;
rc = ENOMEM;
txq = rte_zmalloc_socket("sfc-txq", sizeof(*txq), 0, socket_id);
if (txq == NULL)
goto fail_txq_alloc;
txq_info->txq = txq;
txq->hw_index = sw_index;
txq->evq = evq;
txq->free_thresh =
(tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
SFC_TX_DEFAULT_FREE_THRESH;
txq->flags = tx_conf->txq_flags;
rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries),
socket_id, &txq->mem);
if (rc != 0)
goto fail_dma_alloc;
memset(&info, 0, sizeof(info));
info.free_thresh = txq->free_thresh;
info.flags = tx_conf->txq_flags;
info.txq_entries = txq_info->entries;
info.dma_desc_size_max = encp->enc_tx_dma_desc_size_max;
info.txq_hw_ring = txq->mem.esm_base;
info.evq_entries = txq_info->entries;
info.evq_hw_ring = evq->mem.esm_base;
info.hw_index = txq->hw_index;
info.mem_bar = sa->mem_bar.esb_base;
rc = sa->dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index,
&SFC_DEV_TO_PCI(sa->eth_dev)->addr,
socket_id, &info, &txq->dp);
if (rc != 0)
goto fail_dp_tx_qinit;
evq->dp_txq = txq->dp;
txq->state = SFC_TXQ_INITIALIZED;
txq_info->deferred_start = (tx_conf->tx_deferred_start != 0);
return 0;
fail_dp_tx_qinit:
sfc_dma_free(sa, &txq->mem);
fail_dma_alloc:
txq_info->txq = NULL;
rte_free(txq);
fail_txq_alloc:
sfc_ev_qfini(evq);
fail_ev_qinit:
txq_info->entries = 0;
fail_bad_conf:
sfc_log_init(sa, "failed (TxQ = %u, rc = %d)", sw_index, rc);
return rc;
}
int
sfxge_mcdi_ioctl(struct sfxge_softc *sc, sfxge_ioc_t *ip)
{
const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp);
struct sfxge_mcdi *mp = &(sc->mcdi);
efx_mcdi_req_t emr;
uint8_t *mcdibuf;
int rc;
if (mp->state == SFXGE_MCDI_UNINITIALIZED) {
rc = ENODEV;
goto fail1;
}
if (!(encp->enc_features & EFX_FEATURE_MCDI)) {
rc = ENOTSUP;
goto fail2;
}
if (ip->u.mcdi.len > SFXGE_MCDI_MAX_PAYLOAD) {
rc = EINVAL;
goto fail3;
}
mcdibuf = malloc(SFXGE_MCDI_MAX_PAYLOAD, M_TEMP, M_WAITOK | M_ZERO);
if ((rc = copyin(ip->u.mcdi.payload, mcdibuf, ip->u.mcdi.len)) != 0) {
goto fail5;
}
emr.emr_cmd = ip->u.mcdi.cmd;
emr.emr_in_buf = mcdibuf;
emr.emr_in_length = ip->u.mcdi.len;
emr.emr_out_buf = mcdibuf;
emr.emr_out_length = SFXGE_MCDI_MAX_PAYLOAD;
sfxge_mcdi_execute(sc, &emr);
ip->u.mcdi.rc = emr.emr_rc;
ip->u.mcdi.cmd = emr.emr_cmd;
ip->u.mcdi.len = emr.emr_out_length_used;
if ((rc = copyout(mcdibuf, ip->u.mcdi.payload, ip->u.mcdi.len)) != 0) {
goto fail6;
}
/*
* Helpfully trigger a device reset in response to an MCDI_CMD_REBOOT
* Both ports will see ->emt_exception callbacks on the next MCDI poll
*/
if (ip->u.mcdi.cmd == MC_CMD_REBOOT) {
EFSYS_PROBE(mcdi_ioctl_mc_reboot);
/* sfxge_t->s_state_lock held */
(void) sfxge_schedule_reset(sc);
}
free(mcdibuf, M_TEMP);
return (0);
fail6:
fail5:
free(mcdibuf, M_TEMP);
fail3:
fail2:
fail1:
return (rc);
}
int
sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend,
unsigned int *pkt_descs, size_t *pkt_len)
{
uint8_t *tsoh;
const struct tcp_hdr *th;
efsys_dma_addr_t header_paddr;
uint16_t packet_id;
uint32_t sent_seq;
struct rte_mbuf *m = *in_seg;
size_t nh_off = m->l2_len; /* IP header offset */
size_t tcph_off = m->l2_len + m->l3_len; /* TCP header offset */
size_t header_len = m->l2_len + m->l3_len + m->l4_len;
const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->evq->sa->nic);
idx += SFC_TSO_OPT_DESCS_NUM;
/* Packets which have too big headers should be discarded */
if (unlikely(header_len > SFC_TSOH_STD_LEN))
return EMSGSIZE;
/*
* The TCP header must start at most 208 bytes into the frame.
* If it starts later than this then the NIC won't realise
* it's a TCP packet and TSO edits won't be applied
*/
if (unlikely(tcph_off > encp->enc_tx_tso_tcp_header_offset_limit))
return EMSGSIZE;
header_paddr = rte_pktmbuf_iova(m);
/*
* Sometimes headers may be split across multiple mbufs. In such cases
* we need to glue those pieces and store them in some temporary place.
* Also, packet headers must be contiguous in memory, so that
* they can be referred to with a single DMA descriptor. EF10 has no
* limitations on address boundaries crossing by DMA descriptor data.
*/
if (m->data_len < header_len) {
tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
sfc_tso_prepare_header(tsoh, header_len, in_seg, in_off);
header_paddr = rte_malloc_virt2iova((void *)tsoh);
} else {
if (m->data_len == header_len) {
*in_off = 0;
*in_seg = m->next;
} else {
*in_off = header_len;
}
tsoh = rte_pktmbuf_mtod(m, uint8_t *);
}
/* Handle IP header */
if (m->ol_flags & PKT_TX_IPV4) {
const struct ipv4_hdr *iphe4;
iphe4 = (const struct ipv4_hdr *)(tsoh + nh_off);
rte_memcpy(&packet_id, &iphe4->packet_id, sizeof(uint16_t));
packet_id = rte_be_to_cpu_16(packet_id);
} else if (m->ol_flags & PKT_TX_IPV6) {
packet_id = 0;
} else {
return EINVAL;
}
/* Handle TCP header */
th = (const struct tcp_hdr *)(tsoh + tcph_off);
rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t));
sent_seq = rte_be_to_cpu_32(sent_seq);
efx_tx_qdesc_tso2_create(txq->common, packet_id, 0, sent_seq,
m->tso_segsz,
*pend, EFX_TX_FATSOV2_OPT_NDESCS);
*pend += EFX_TX_FATSOV2_OPT_NDESCS;
*pkt_descs += EFX_TX_FATSOV2_OPT_NDESCS;
efx_tx_qdesc_dma_create(txq->common, header_paddr, header_len,
B_FALSE, (*pend)++);
(*pkt_descs)++;
*pkt_len -= header_len;
return 0;
}
