void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index,
unsigned int len, bool checksummed, bool discard)
{
struct efx_nic *efx = rx_queue->efx;
struct efx_rx_buffer *rx_buf;
bool leak_packet = false;
rx_buf = efx_rx_buffer(rx_queue, index);
EFX_BUG_ON_PARANOID(!rx_buf->data);
EFX_BUG_ON_PARANOID(rx_buf->skb && rx_buf->page);
EFX_BUG_ON_PARANOID(!(rx_buf->skb || rx_buf->page));
/* This allows the refill path to post another buffer.
* EFX_RXD_HEAD_ROOM ensures that the slot we are using
* isn't overwritten yet.
*/
rx_queue->removed_count++;
/* Validate the length encoded in the event vs the descriptor pushed */
efx_rx_packet__check_len(rx_queue, rx_buf, len,
&discard, &leak_packet);
EFX_TRACE(efx, "RX queue %d received id %x at %llx+%x %s%s\n",
rx_queue->queue, index,
(unsigned long long)rx_buf->dma_addr, len,
(checksummed ? " [SUMMED]" : ""),
(discard ? " [DISCARD]" : ""));
/* Discard packet, if instructed to do so */
if (unlikely(discard)) {
if (unlikely(leak_packet))
rx_queue->channel->n_skbuff_leaks++;
else
/* We haven't called efx_unmap_rx_buffer yet,
* so fini the entire rx_buffer here */
efx_fini_rx_buffer(rx_queue, rx_buf);
return;
}
/* Release card resources - assumes all RX buffers consumed in-order
* per RX queue
*/
efx_unmap_rx_buffer(efx, rx_buf);
/* Prefetch nice and early so data will (hopefully) be in cache by
* the time we look at it.
*/
prefetch(rx_buf->data);
/* Pipeline receives so that we give time for packet headers to be
* prefetched into cache.
*/
rx_buf->len = len;
if (rx_queue->channel->rx_pkt)
__efx_rx_packet(rx_queue->channel,
rx_queue->channel->rx_pkt,
rx_queue->channel->rx_pkt_csummed);
rx_queue->channel->rx_pkt = rx_buf;
rx_queue->channel->rx_pkt_csummed = checksummed;
}
/* Pass a received packet up through the generic LRO stack
*
* Handles driverlink veto, and passes the fragment up via
* the appropriate LRO method
*/
static void efx_rx_packet_lro(struct efx_channel *channel,
struct efx_rx_buffer *rx_buf)
{
struct net_lro_mgr *lro_mgr = &channel->lro_mgr;
void *priv = channel;
/* Pass the skb/page into the LRO engine */
if (rx_buf->page) {
struct skb_frag_struct frags;
frags.page = rx_buf->page;
frags.page_offset = efx_rx_buf_offset(rx_buf);
frags.size = rx_buf->len;
lro_receive_frags(lro_mgr, &frags, rx_buf->len,
rx_buf->len, priv, 0);
EFX_BUG_ON_PARANOID(rx_buf->skb);
rx_buf->page = NULL;
} else {
EFX_BUG_ON_PARANOID(!rx_buf->skb);
lro_receive_skb(lro_mgr, rx_buf->skb, priv);
rx_buf->skb = NULL;
}
}
/*
* Verify that our various assumptions about sk_buffs and the conditions
* under which TSO will be attempted hold true. Return the protocol number.
*/
static __be16 efx_tso_check_protocol(struct sk_buff *skb)
{
__be16 protocol = skb->protocol;
EFX_BUG_ON_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
protocol);
if (protocol == htons(ETH_P_8021Q)) {
/* Find the encapsulated protocol; reset network header
* and transport header based on that. */
struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
protocol = veh->h_vlan_encapsulated_proto;
skb_set_network_header(skb, sizeof(*veh));
if (protocol == htons(ETH_P_IP))
skb_set_transport_header(skb, sizeof(*veh) +
4 * ip_hdr(skb)->ihl);
else if (protocol == htons(ETH_P_IPV6))
skb_set_transport_header(skb, sizeof(*veh) +
sizeof(struct ipv6hdr));
}
if (protocol == htons(ETH_P_IP)) {
EFX_BUG_ON_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP);
} else {
EFX_BUG_ON_PARANOID(protocol != htons(ETH_P_IPV6));
EFX_BUG_ON_PARANOID(ipv6_hdr(skb)->nexthdr != NEXTHDR_TCP);
}
EFX_BUG_ON_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data)
+ (tcp_hdr(skb)->doff << 2u)) >
skb_headlen(skb));
return protocol;
}
void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
{
unsigned fill_level;
struct efx_nic *efx = tx_queue->efx;
EFX_BUG_ON_PARANOID(index > tx_queue->ptr_mask);
efx_dequeue_buffers(tx_queue, index);
/* See if we need to restart the netif queue. This barrier
* separates the update of read_count from the test of the
* queue state. */
smp_mb();
if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&
likely(efx->port_enabled)) {
fill_level = tx_queue->insert_count - tx_queue->read_count;
if (fill_level < EFX_TXQ_THRESHOLD(efx)) {
EFX_BUG_ON_PARANOID(!efx_dev_registered(efx));
netif_tx_wake_queue(tx_queue->core_txq);
}
}
/* Check whether the hardware queue is now empty */
if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {
tx_queue->old_write_count = ACCESS_ONCE(tx_queue->write_count);
if (tx_queue->read_count == tx_queue->old_write_count) {
smp_mb();
tx_queue->empty_read_count =
tx_queue->read_count | EFX_EMPTY_COUNT_VALID;
}
}
}
void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
{
unsigned fill_level;
struct efx_nic *efx = tx_queue->efx;
EFX_BUG_ON_PARANOID(index > efx->type->txd_ring_mask);
efx_dequeue_buffers(tx_queue, index);
/* See if we need to restart the netif queue. This barrier
* separates the update of read_count from the test of
* stopped. */
smp_mb();
if (unlikely(tx_queue->stopped) && likely(efx->port_enabled)) {
fill_level = tx_queue->insert_count - tx_queue->read_count;
if (fill_level < EFX_NETDEV_TX_THRESHOLD(tx_queue)) {
EFX_BUG_ON_PARANOID(!efx_dev_registered(efx));
/* Do this under netif_tx_lock(), to avoid racing
* with efx_xmit(). */
netif_tx_lock(efx->net_dev);
if (tx_queue->stopped) {
tx_queue->stopped = 0;
efx_wake_queue(efx);
}
netif_tx_unlock(efx->net_dev);
}
}
}
/**
* tso_fill_packet_with_fragment - form descriptors for the current fragment
* @tx_queue: Efx TX queue
* @skb: Socket buffer
* @st: TSO state
*
* Form descriptors for the current fragment, until we reach the end
* of fragment or end-of-packet. Return 0 on success, 1 if not enough
* space in @tx_queue.
*/
static inline int tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue,
const struct sk_buff *skb,
struct tso_state *st)
{
int n, end_of_packet, rc;
if (st->ifc.len == 0)
return 0;
if (st->packet_space == 0)
return 0;
EFX_BUG_ON_PARANOID(st->ifc.len <= 0);
EFX_BUG_ON_PARANOID(st->packet_space <= 0);
n = min(st->ifc.len, st->packet_space);
st->packet_space -= n;
st->remaining_len -= n;
st->ifc.len -= n;
st->ifc.page_off += n;
end_of_packet = st->remaining_len == 0 || st->packet_space == 0;
rc = efx_tx_queue_insert(tx_queue, st->ifc.dma_addr, n,
st->remaining_len ? NULL : skb,
end_of_packet, st->ifc.unmap_addr,
st->ifc.len ? 0 : st->ifc.unmap_len);
st->ifc.dma_addr += n;
return rc;
}
static inline void i2c_release(struct efx_i2c_interface *i2c)
{
EFX_WARN_ON_PARANOID(!i2c->scl);
EFX_WARN_ON_PARANOID(!i2c->sda);
/* Devices may time out if operations do not end */
setscl(i2c, 1);
setsda(i2c, 1);
EFX_BUG_ON_PARANOID(getsda(i2c) != 1);
EFX_BUG_ON_PARANOID(getscl(i2c) != 1);
}
/*
* Verify that our various assumptions about sk_buffs and the conditions
* under which TSO will be attempted hold true.
*/
static inline void efx_tso_check_safe(const struct sk_buff *skb)
{
EFX_BUG_ON_PARANOID(skb->protocol != htons(ETH_P_IP));
EFX_BUG_ON_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
skb->protocol);
EFX_BUG_ON_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP);
EFX_BUG_ON_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data)
+ (tcp_hdr(skb)->doff << 2u)) >
skb_headlen(skb));
}
/* Handle a received packet. Second half: Touches packet payload. */
void __efx_rx_packet(struct efx_channel *channel,
struct efx_rx_buffer *rx_buf, bool checksummed)
{
struct efx_nic *efx = channel->efx;
struct sk_buff *skb;
/* If we're in loopback test, then pass the packet directly to the
* loopback layer, and free the rx_buf here
*/
if (unlikely(efx->loopback_selftest)) {
efx_loopback_rx_packet(efx, rx_buf->data, rx_buf->len);
efx_free_rx_buffer(efx, rx_buf);
goto done;
}
if (rx_buf->skb) {
prefetch(skb_shinfo(rx_buf->skb));
skb_put(rx_buf->skb, rx_buf->len);
/* Move past the ethernet header. rx_buf->data still points
* at the ethernet header */
rx_buf->skb->protocol = eth_type_trans(rx_buf->skb,
efx->net_dev);
}
if (likely(checksummed || rx_buf->page)) {
efx_rx_packet_lro(channel, rx_buf);
goto done;
}
/* We now own the SKB */
skb = rx_buf->skb;
rx_buf->skb = NULL;
EFX_BUG_ON_PARANOID(rx_buf->page);
EFX_BUG_ON_PARANOID(rx_buf->skb);
EFX_BUG_ON_PARANOID(!skb);
/* Set the SKB flags */
skb->ip_summed = CHECKSUM_NONE;
skb_record_rx_queue(skb, channel->channel);
/* Pass the packet up */
netif_receive_skb(skb);
/* Update allocation strategy method */
channel->rx_alloc_level += RX_ALLOC_FACTOR_SKB;
done:
;
}
static inline struct efx_tx_buffer *
efx_tx_queue_get_insert_buffer(const struct efx_tx_queue *tx_queue)
{
struct efx_tx_buffer *buffer =
__efx_tx_queue_get_insert_buffer(tx_queue);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(buffer->flags);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
return buffer;
}
/* Pass a received packet up through the generic LRO stack
*
* Handles driverlink veto, and passes the fragment up via
* the appropriate LRO method
*/
static void efx_rx_packet_lro(struct efx_channel *channel,
struct efx_rx_buffer *rx_buf,
bool checksummed)
{
struct napi_struct *napi = &channel->napi_str;
gro_result_t gro_result;
/* Pass the skb/page into the LRO engine */
if (rx_buf->page) {
struct page *page = rx_buf->page;
struct sk_buff *skb;
EFX_BUG_ON_PARANOID(rx_buf->skb);
rx_buf->page = NULL;
skb = napi_get_frags(napi);
if (!skb) {
put_page(page);
return;
}
skb_shinfo(skb)->frags[0].page = page;
skb_shinfo(skb)->frags[0].page_offset =
efx_rx_buf_offset(rx_buf);
skb_shinfo(skb)->frags[0].size = rx_buf->len;
skb_shinfo(skb)->nr_frags = 1;
skb->len = rx_buf->len;
skb->data_len = rx_buf->len;
skb->truesize += rx_buf->len;
skb->ip_summed =
checksummed ? CHECKSUM_UNNECESSARY : CHECKSUM_NONE;
skb_record_rx_queue(skb, channel->channel);
gro_result = napi_gro_frags(napi);
} else {
struct sk_buff *skb = rx_buf->skb;
EFX_BUG_ON_PARANOID(!skb);
EFX_BUG_ON_PARANOID(!checksummed);
rx_buf->skb = NULL;
gro_result = napi_gro_receive(napi, skb);
}
if (gro_result == GRO_NORMAL) {
channel->rx_alloc_level += RX_ALLOC_FACTOR_SKB;
} else if (gro_result != GRO_DROP) {
channel->rx_alloc_level += RX_ALLOC_FACTOR_LRO;
channel->irq_mod_score += 2;
}
}
/**
* efx_fast_push_rx_descriptors - push new RX descriptors quickly
* @rx_queue: RX descriptor queue
*
* This will aim to fill the RX descriptor queue up to
* @rx_queue->@max_fill. If there is insufficient atomic
* memory to do so, a slow fill will be scheduled.
*
* The caller must provide serialisation (none is used here). In practise,
* this means this function must run from the NAPI handler, or be called
* when NAPI is disabled.
*/
void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue, bool atomic)
{
struct efx_nic *efx = rx_queue->efx;
unsigned int fill_level, batch_size;
int space, rc = 0;
if (!rx_queue->refill_enabled)
return;
/* Calculate current fill level, and exit if we don't need to fill */
fill_level = (rx_queue->added_count - rx_queue->removed_count);
EFX_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries);
if (fill_level >= rx_queue->fast_fill_trigger)
goto out;
/* Record minimum fill level */
if (unlikely(fill_level < rx_queue->min_fill)) {
if (fill_level)
rx_queue->min_fill = fill_level;
}
batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page;
space = rx_queue->max_fill - fill_level;
EFX_BUG_ON_PARANOID(space < batch_size);
netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
"RX queue %d fast-filling descriptor ring from"
" level %d to level %d\n",
efx_rx_queue_index(rx_queue), fill_level,
rx_queue->max_fill);
do {
rc = efx_init_rx_buffers(rx_queue, atomic);
if (unlikely(rc)) {
/* Ensure that we don't leave the rx queue empty */
if (rx_queue->added_count == rx_queue->removed_count)
efx_schedule_slow_fill(rx_queue);
goto out;
}
} while ((space -= batch_size) >= batch_size);
netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
"RX queue %d fast-filled descriptor ring "
"to level %d\n", efx_rx_queue_index(rx_queue),
rx_queue->added_count - rx_queue->removed_count);
out:
if (rx_queue->notified_count != rx_queue->added_count)
efx_nic_notify_rx_desc(rx_queue);
}
int efx_probe_rx_queue(struct efx_rx_queue *rx_queue)
{
struct efx_nic *efx = rx_queue->efx;
unsigned int entries;
int rc;
/* Create the smallest power-of-two aligned ring */
entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE);
EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
rx_queue->ptr_mask = entries - 1;
netif_dbg(efx, probe, efx->net_dev,
"creating RX queue %d size %#x mask %#x\n",
efx_rx_queue_index(rx_queue), efx->rxq_entries,
rx_queue->ptr_mask);
/* Allocate RX buffers */
rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),
GFP_KERNEL);
if (!rx_queue->buffer)
return -ENOMEM;
rc = efx_nic_probe_rx(rx_queue);
if (rc) {
kfree(rx_queue->buffer);
rx_queue->buffer = NULL;
}
return rc;
}
/* Remove descriptors put into a tx_queue. */
static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
{
struct efx_tx_buffer *buffer;
dma_addr_t unmap_addr;
/* Work backwards until we hit the original insert pointer value */
while (tx_queue->insert_count != tx_queue->write_count) {
--tx_queue->insert_count;
buffer = &tx_queue->buffer[tx_queue->insert_count &
tx_queue->ptr_mask];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->skb);
if (buffer->unmap_len) {
unmap_addr = (buffer->dma_addr + buffer->len -
buffer->unmap_len);
if (buffer->unmap_single)
pci_unmap_single(tx_queue->efx->pci_dev,
unmap_addr, buffer->unmap_len,
PCI_DMA_TODEVICE);
else
pci_unmap_page(tx_queue->efx->pci_dev,
unmap_addr, buffer->unmap_len,
PCI_DMA_TODEVICE);
buffer->unmap_len = 0;
}
buffer->len = 0;
buffer->continuation = true;
}
}
/*
* Allocate a page worth of efx_tso_header structures, and string them
* into the tx_queue->tso_headers_free linked list. Return 0 or -ENOMEM.
*/
static int efx_tsoh_block_alloc(struct efx_tx_queue *tx_queue)
{
struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
struct efx_tso_header *tsoh;
dma_addr_t dma_addr;
u8 *base_kva, *kva;
base_kva = pci_alloc_consistent(pci_dev, PAGE_SIZE, &dma_addr);
if (base_kva == NULL) {
netif_err(tx_queue->efx, tx_err, tx_queue->efx->net_dev,
"Unable to allocate page for TSO headers\n");
return -ENOMEM;
}
/* pci_alloc_consistent() allocates pages. */
EFX_BUG_ON_PARANOID(dma_addr & (PAGE_SIZE - 1u));
for (kva = base_kva; kva < base_kva + PAGE_SIZE; kva += TSOH_STD_SIZE) {
tsoh = (struct efx_tso_header *)kva;
tsoh->dma_addr = dma_addr + (TSOH_BUFFER(tsoh) - base_kva);
tsoh->next = tx_queue->tso_headers_free;
tx_queue->tso_headers_free = tsoh;
}
return 0;
}
int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)
{
struct efx_nic *efx = tx_queue->efx;
unsigned int entries;
int i, rc;
/* Create the smallest power-of-two aligned ring */
entries = max(roundup_pow_of_two(efx->txq_entries), EFX_MIN_DMAQ_SIZE);
EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
tx_queue->ptr_mask = entries - 1;
netif_dbg(efx, probe, efx->net_dev,
"creating TX queue %d size %#x mask %#x\n",
tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask);
/* Allocate software ring */
tx_queue->buffer = kzalloc(entries * sizeof(*tx_queue->buffer),
GFP_KERNEL);
if (!tx_queue->buffer)
return -ENOMEM;
for (i = 0; i <= tx_queue->ptr_mask; ++i)
tx_queue->buffer[i].continuation = true;
/* Allocate hardware ring */
rc = efx_nic_probe_tx(tx_queue);
if (rc)
goto fail;
return 0;
fail:
kfree(tx_queue->buffer);
tx_queue->buffer = NULL;
return rc;
}
static inline struct efx_mcdi_iface *efx_mcdi(struct efx_nic *efx)
{
struct siena_nic_data *nic_data;
EFX_BUG_ON_PARANOID(efx_nic_rev(efx) < EFX_REV_SIENA_A0);
nic_data = efx->nic_data;
return &nic_data->mcdi;
}
/* Allocate and construct an SKB around a struct page.*/
static struct sk_buff *efx_rx_mk_skb(struct efx_rx_buffer *rx_buf,
struct efx_nic *efx,
int hdr_len)
{
struct sk_buff *skb;
/* Allocate an SKB to store the headers */
skb = netdev_alloc_skb(efx->net_dev, hdr_len + EFX_PAGE_SKB_ALIGN);
if (unlikely(skb == NULL)) {
EFX_ERR_RL(efx, "RX out of memory for skb\n");
return NULL;
}
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags);
EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len);
skb->ip_summed = CHECKSUM_UNNECESSARY;
skb_reserve(skb, EFX_PAGE_SKB_ALIGN);
skb->len = rx_buf->len;
skb->truesize = rx_buf->len + sizeof(struct sk_buff);
memcpy(skb->data, rx_buf->data, hdr_len);
skb->tail += hdr_len;
/* Append the remaining page onto the frag list */
if (unlikely(rx_buf->len > hdr_len)) {
struct skb_frag_struct *frag = skb_shinfo(skb)->frags;
frag->page = rx_buf->page;
frag->page_offset = efx_rx_buf_offset(rx_buf) + hdr_len;
frag->size = skb->len - hdr_len;
skb_shinfo(skb)->nr_frags = 1;
skb->data_len = frag->size;
} else {
__free_pages(rx_buf->page, efx->rx_buffer_order);
skb->data_len = 0;
}
/* Ownership has transferred from the rx_buf to skb */
rx_buf->page = NULL;
/* Move past the ethernet header */
skb->protocol = eth_type_trans(skb, efx->net_dev);
return skb;
}
/*
* Put a TSO header into the TX queue.
*
* This is special-cased because we know that it is small enough to fit in
* a single fragment, and we know it doesn't cross a page boundary. It
* also allows us to not worry about end-of-packet etc.
*/
static void efx_tso_put_header(struct efx_tx_queue *tx_queue,
struct efx_tso_header *tsoh, unsigned len)
{
struct efx_tx_buffer *buffer;
buffer = &tx_queue->buffer[tx_queue->insert_count & tx_queue->ptr_mask];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(!buffer->continuation);
EFX_BUG_ON_PARANOID(buffer->tsoh);
buffer->len = len;
buffer->dma_addr = tsoh->dma_addr;
buffer->tsoh = tsoh;
++tx_queue->insert_count;
}
/* Allocate and construct an SKB around page fragments */
static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel,
struct efx_rx_buffer *rx_buf,
unsigned int n_frags,
u8 *eh, int hdr_len)
{
struct efx_nic *efx = channel->efx;
struct sk_buff *skb;
/* Allocate an SKB to store the headers */
skb = netdev_alloc_skb(efx->net_dev,
efx->rx_ip_align + efx->rx_prefix_size +
hdr_len);
if (unlikely(skb == NULL)) {
atomic_inc(&efx->n_rx_noskb_drops);
return NULL;
}
EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len);
memcpy(skb->data + efx->rx_ip_align, eh - efx->rx_prefix_size,
efx->rx_prefix_size + hdr_len);
skb_reserve(skb, efx->rx_ip_align + efx->rx_prefix_size);
__skb_put(skb, hdr_len);
/* Append the remaining page(s) onto the frag list */
if (rx_buf->len > hdr_len) {
rx_buf->page_offset += hdr_len;
rx_buf->len -= hdr_len;
for (;;) {
skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
rx_buf->page, rx_buf->page_offset,
rx_buf->len);
rx_buf->page = NULL;
skb->len += rx_buf->len;
skb->data_len += rx_buf->len;
if (skb_shinfo(skb)->nr_frags == n_frags)
break;
rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
}
} else {
__free_pages(rx_buf->page, efx->rx_buffer_order);
rx_buf->page = NULL;
n_frags = 0;
}
skb->truesize += n_frags * efx->rx_buffer_truesize;
/* Move past the ethernet header */
skb->protocol = eth_type_trans(skb, efx->net_dev);
skb_mark_napi_id(skb, &channel->napi_str);
return skb;
}
/* Parse the SKB header and initialise state. */
static inline void tso_start(struct tso_state *st, const struct sk_buff *skb)
{
/* All ethernet/IP/TCP headers combined size is TCP header size
* plus offset of TCP header relative to start of packet.
*/
st->p.header_length = ((tcp_hdr(skb)->doff << 2u)
+ PTR_DIFF(tcp_hdr(skb), skb->data));
st->p.full_packet_size = (st->p.header_length
+ skb_shinfo(skb)->gso_size);
st->p.ipv4_id = ntohs(ip_hdr(skb)->id);
st->seqnum = ntohl(tcp_hdr(skb)->seq);
EFX_BUG_ON_PARANOID(tcp_hdr(skb)->urg);
EFX_BUG_ON_PARANOID(tcp_hdr(skb)->syn);
EFX_BUG_ON_PARANOID(tcp_hdr(skb)->rst);
st->packet_space = st->p.full_packet_size;
st->remaining_len = skb->len - st->p.header_length;
}
/**
* tso_fill_packet_with_fragment - form descriptors for the current fragment
* @tx_queue: Efx TX queue
* @skb: Socket buffer
* @st: TSO state
*
* Form descriptors for the current fragment, until we reach the end
* of fragment or end-of-packet. Return 0 on success, 1 if not enough
* space in @tx_queue.
*/
static int tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue,
const struct sk_buff *skb,
struct tso_state *st)
{
struct efx_tx_buffer *buffer;
int n, end_of_packet, rc;
if (st->in_len == 0)
return 0;
if (st->packet_space == 0)
return 0;
EFX_BUG_ON_PARANOID(st->in_len <= 0);
EFX_BUG_ON_PARANOID(st->packet_space <= 0);
n = min(st->in_len, st->packet_space);
st->packet_space -= n;
st->out_len -= n;
st->in_len -= n;
rc = efx_tx_queue_insert(tx_queue, st->dma_addr, n, &buffer);
if (likely(rc == 0)) {
if (st->out_len == 0)
/* Transfer ownership of the skb */
buffer->skb = skb;
end_of_packet = st->out_len == 0 || st->packet_space == 0;
buffer->continuation = !end_of_packet;
if (st->in_len == 0) {
/* Transfer ownership of the pci mapping */
buffer->unmap_len = st->unmap_len;
buffer->unmap_single = st->unmap_single;
st->unmap_len = 0;
}
}
st->dma_addr += n;
return rc;
}
/* Parse the SKB header and initialise state. */
static void tso_start(struct tso_state *st, const struct sk_buff *skb)
{
/* All ethernet/IP/TCP headers combined size is TCP header size
* plus offset of TCP header relative to start of packet.
*/
st->header_len = ((tcp_hdr(skb)->doff << 2u)
+ PTR_DIFF(tcp_hdr(skb), skb->data));
st->full_packet_size = st->header_len + skb_shinfo(skb)->gso_size;
st->ipv4_id = ntohs(ip_hdr(skb)->id);
st->seqnum = ntohl(tcp_hdr(skb)->seq);
EFX_BUG_ON_PARANOID(tcp_hdr(skb)->urg);
EFX_BUG_ON_PARANOID(tcp_hdr(skb)->syn);
EFX_BUG_ON_PARANOID(tcp_hdr(skb)->rst);
st->packet_space = st->full_packet_size;
st->out_len = skb->len - st->header_len;
st->unmap_len = 0;
st->unmap_single = false;
}
/* Pass a received packet up through the generic LRO stack
*
* Handles driverlink veto, and passes the fragment up via
* the appropriate LRO method
*/
static void efx_rx_packet_lro(struct efx_channel *channel,
struct efx_rx_buffer *rx_buf)
{
struct napi_struct *napi = &channel->napi_str;
/* Pass the skb/page into the LRO engine */
if (rx_buf->page) {
struct sk_buff *skb = napi_get_frags(napi);
if (!skb) {
put_page(rx_buf->page);
goto out;
}
skb_shinfo(skb)->frags[0].page = rx_buf->page;
skb_shinfo(skb)->frags[0].page_offset =
efx_rx_buf_offset(rx_buf);
skb_shinfo(skb)->frags[0].size = rx_buf->len;
skb_shinfo(skb)->nr_frags = 1;
skb->len = rx_buf->len;
skb->data_len = rx_buf->len;
skb->truesize += rx_buf->len;
skb->ip_summed = CHECKSUM_UNNECESSARY;
napi_gro_frags(napi);
out:
EFX_BUG_ON_PARANOID(rx_buf->skb);
rx_buf->page = NULL;
} else {
EFX_BUG_ON_PARANOID(!rx_buf->skb);
napi_gro_receive(napi, rx_buf->skb);
rx_buf->skb = NULL;
}
}
static void efx_unmap_rx_buffer(struct efx_nic *efx,
struct efx_rx_buffer *rx_buf)
{
if (rx_buf->page) {
EFX_BUG_ON_PARANOID(rx_buf->skb);
if (rx_buf->unmap_addr) {
pci_unmap_page(efx->pci_dev, rx_buf->unmap_addr,
efx_rx_buf_size(efx),
PCI_DMA_FROMDEVICE);
rx_buf->unmap_addr = 0;
}
} else if (likely(rx_buf->skb)) {
pci_unmap_single(efx->pci_dev, rx_buf->dma_addr,
rx_buf->len, PCI_DMA_FROMDEVICE);
}
}
static void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev)
{
unsigned int monitor, state, value;
const char *name, *state_txt;
monitor = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_MONITOR);
state = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_STATE);
value = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_VALUE);
/* Deal gracefully with the board having more drivers than we
* know about, but do not expect new sensor states. */
name = (monitor >= ARRAY_SIZE(sensor_names))
? "No sensor name available" :
sensor_names[monitor];
EFX_BUG_ON_PARANOID(state >= ARRAY_SIZE(sensor_status_names));
state_txt = sensor_status_names[state];
netif_err(efx, hw, efx->net_dev,
"Sensor %d (%s) reports condition '%s' for raw value %d\n",
monitor, name, state_txt, value);
}
void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev)
{
unsigned int type, state, value;
const char *name = NULL, *state_txt;
type = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_MONITOR);
state = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_STATE);
value = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_VALUE);
if (type < ARRAY_SIZE(efx_mcdi_sensor_type))
name = efx_mcdi_sensor_type[type].label;
if (!name)
name = "No sensor name available";
EFX_BUG_ON_PARANOID(state >= ARRAY_SIZE(sensor_status_names));
state_txt = sensor_status_names[state];
netif_err(efx, hw, efx->net_dev,
"Sensor %d (%s) reports condition '%s' for raw value %d\n",
type, name, state_txt, value);
}
static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1)
{
/* We need to consider both queues that the net core sees as one */
struct efx_tx_queue *txq2 = efx_tx_queue_partner(txq1);
struct efx_nic *efx = txq1->efx;
unsigned int fill_level;
fill_level = max(txq1->insert_count - txq1->old_read_count,
txq2->insert_count - txq2->old_read_count);
if (likely(fill_level < efx->txq_stop_thresh))
return;
/* We used the stale old_read_count above, which gives us a
* pessimistic estimate of the fill level (which may even
* validly be >= efx->txq_entries). Now try again using
* read_count (more likely to be a cache miss).
*
* If we read read_count and then conditionally stop the
* queue, it is possible for the completion path to race with
* us and complete all outstanding descriptors in the middle,
* after which there will be no more completions to wake it.
* Therefore we stop the queue first, then read read_count
* (with a memory barrier to ensure the ordering), then
* restart the queue if the fill level turns out to be low
* enough.
*/
netif_tx_stop_queue(txq1->core_txq);
smp_mb();
txq1->old_read_count = ACCESS_ONCE(txq1->read_count);
txq2->old_read_count = ACCESS_ONCE(txq2->read_count);
fill_level = max(txq1->insert_count - txq1->old_read_count,
txq2->insert_count - txq2->old_read_count);
EFX_BUG_ON_PARANOID(fill_level >= efx->txq_entries);
if (likely(fill_level < efx->txq_stop_thresh)) {
smp_mb();
if (likely(!efx->loopback_selftest))
netif_tx_start_queue(txq1->core_txq);
}
}
static void efx_mcdi_process_link_change(struct efx_nic *efx, efx_qword_t *ev)
{
u32 flags, fcntl, speed, lpa;
speed = EFX_QWORD_FIELD(*ev, MCDI_EVENT_LINKCHANGE_SPEED);
EFX_BUG_ON_PARANOID(speed >= ARRAY_SIZE(efx_mcdi_event_link_speed));
speed = efx_mcdi_event_link_speed[speed];
flags = EFX_QWORD_FIELD(*ev, MCDI_EVENT_LINKCHANGE_LINK_FLAGS);
fcntl = EFX_QWORD_FIELD(*ev, MCDI_EVENT_LINKCHANGE_FCNTL);
lpa = EFX_QWORD_FIELD(*ev, MCDI_EVENT_LINKCHANGE_LP_CAP);
/* efx->link_state is only modified by efx_mcdi_phy_get_link(),
* which is only run after flushing the event queues. Therefore, it
* is safe to modify the link state outside of the mac_lock here.
*/
efx_mcdi_phy_decode_link(efx, &efx->link_state, speed, flags, fcntl);
efx_mcdi_phy_check_fcntl(efx, lpa);
efx_link_status_changed(efx);
}
static void siena_mcdi_request(struct efx_nic *efx,
const efx_dword_t *hdr, size_t hdr_len,
const efx_dword_t *sdu, size_t sdu_len)
{
unsigned pdu = FR_CZ_MC_TREG_SMEM + MCDI_PDU(efx);
unsigned doorbell = FR_CZ_MC_TREG_SMEM + MCDI_DOORBELL(efx);
unsigned int i;
unsigned int inlen_dw = DIV_ROUND_UP(sdu_len, 4);
EFX_BUG_ON_PARANOID(hdr_len != 4);
efx_writed(efx, hdr, pdu);
for (i = 0; i < inlen_dw; i++)
efx_writed(efx, &sdu[i], pdu + hdr_len + 4 * i);
/* Ensure the request is written out before the doorbell */
wmb();
/* ring the doorbell with a distinctive value */
_efx_writed(efx, (__force __le32) 0x45789abc, doorbell);
}