/* Attempt to recycle the page if there is an RX recycle ring; the page can * only be added if this is the final RX buffer, to prevent pages being used in * the descriptor ring and appearing in the recycle ring simultaneously. */ static void efx_recycle_rx_page(struct efx_channel *channel, struct efx_rx_buffer *rx_buf) { struct page *page = rx_buf->page; struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); struct efx_nic *efx = rx_queue->efx; unsigned index; /* Only recycle the page after processing the final buffer. */ if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE)) return; index = rx_queue->page_add & rx_queue->page_ptr_mask; if (rx_queue->page_ring[index] == NULL) { unsigned read_index = rx_queue->page_remove & rx_queue->page_ptr_mask; /* The next slot in the recycle ring is available, but * increment page_remove if the read pointer currently * points here. */ if (read_index == index) ++rx_queue->page_remove; rx_queue->page_ring[index] = page; ++rx_queue->page_add; return; } ++rx_queue->page_recycle_full; efx_unmap_rx_buffer(efx, rx_buf); put_page(rx_buf->page); }
static void efx_rx_deliver(struct efx_channel *channel, u8 *eh, struct efx_rx_buffer *rx_buf, unsigned int n_frags) { struct sk_buff *skb; u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS); skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len); if (unlikely(skb == NULL)) { struct efx_rx_queue *rx_queue; rx_queue = efx_channel_get_rx_queue(channel); efx_free_rx_buffers(rx_queue, rx_buf, n_frags); return; } skb_record_rx_queue(skb, channel->rx_queue.core_index); /* Set the SKB flags */ skb_checksum_none_assert(skb); if (likely(rx_buf->flags & EFX_RX_PKT_CSUMMED)) skb->ip_summed = CHECKSUM_UNNECESSARY; efx_rx_skb_attach_timestamp(channel, skb); if (channel->type->receive_skb) if (channel->type->receive_skb(channel, skb)) return; /* Pass the packet up */ netif_receive_skb(skb); }
static void efx_discard_rx_packet(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, unsigned int n_frags) { struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); efx_recycle_rx_pages(channel, rx_buf, n_frags); efx_free_rx_buffers(rx_queue, rx_buf, n_frags); }
/* Recycle the pages that are used by buffers that have just been received. */ static void efx_recycle_rx_pages(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, unsigned int n_frags) { struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); do { efx_recycle_rx_page(channel, rx_buf); rx_buf = efx_rx_buf_next(rx_queue, rx_buf); } while (--n_frags); }
/* Pass a received packet up through GRO. GRO can handle pages * regardless of checksum state and skbs with a good checksum. */ static void efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, unsigned int n_frags, u8 *eh) { struct napi_struct *napi = &channel->napi_str; gro_result_t gro_result; struct efx_nic *efx = channel->efx; struct sk_buff *skb; skb = napi_get_frags(napi); if (unlikely(!skb)) { struct efx_rx_queue *rx_queue; rx_queue = efx_channel_get_rx_queue(channel); efx_free_rx_buffers(rx_queue, rx_buf, n_frags); return; } if (efx->net_dev->features & NETIF_F_RXHASH) skb_set_hash(skb, efx_rx_buf_hash(efx, eh), PKT_HASH_TYPE_L3); skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ? CHECKSUM_UNNECESSARY : CHECKSUM_NONE); for (;;) { skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, rx_buf->page, rx_buf->page_offset, rx_buf->len); rx_buf->page = NULL; skb->len += rx_buf->len; if (skb_shinfo(skb)->nr_frags == n_frags) break; rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); } skb->data_len = skb->len; skb->truesize += n_frags * efx->rx_buffer_truesize; skb_record_rx_queue(skb, channel->rx_queue.core_index); skb_mark_napi_id(skb, &channel->napi_str); gro_result = napi_gro_frags(napi); if (gro_result != GRO_DROP) channel->irq_mod_score += 2; }
/* Handle a received packet. Second half: Touches packet payload. */ void __efx_rx_packet(struct efx_channel *channel) { struct efx_nic *efx = channel->efx; struct efx_rx_buffer *rx_buf = efx_rx_buffer(&channel->rx_queue, channel->rx_pkt_index); u8 *eh = efx_rx_buf_va(rx_buf); /* Read length from the prefix if necessary. This already * excludes the length of the prefix itself. */ if (rx_buf->flags & EFX_RX_PKT_PREFIX_LEN) rx_buf->len = le16_to_cpup((__le16 *) (eh + efx->rx_packet_len_offset)); /* If we're in loopback test, then pass the packet directly to the * loopback layer, and free the rx_buf here */ if (unlikely(efx->loopback_selftest)) { struct efx_rx_queue *rx_queue; efx_loopback_rx_packet(efx, eh, rx_buf->len); rx_queue = efx_channel_get_rx_queue(channel); efx_free_rx_buffers(rx_queue, rx_buf, channel->rx_pkt_n_frags); goto out; } if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM))) rx_buf->flags &= ~EFX_RX_PKT_CSUMMED; if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb && !efx_channel_busy_polling(channel)) efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh); else efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags); out: channel->rx_pkt_n_frags = 0; }