void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index,
unsigned int len, bool checksummed, bool discard)
{
struct efx_nic *efx = rx_queue->efx;
struct efx_rx_buffer *rx_buf;
bool leak_packet = false;
rx_buf = efx_rx_buffer(rx_queue, index);
EFX_BUG_ON_PARANOID(!rx_buf->data);
EFX_BUG_ON_PARANOID(rx_buf->skb && rx_buf->page);
EFX_BUG_ON_PARANOID(!(rx_buf->skb || rx_buf->page));
/* This allows the refill path to post another buffer.
* EFX_RXD_HEAD_ROOM ensures that the slot we are using
* isn't overwritten yet.
*/
rx_queue->removed_count++;
/* Validate the length encoded in the event vs the descriptor pushed */
efx_rx_packet__check_len(rx_queue, rx_buf, len,
&discard, &leak_packet);
EFX_TRACE(efx, "RX queue %d received id %x at %llx+%x %s%s\n",
rx_queue->queue, index,
(unsigned long long)rx_buf->dma_addr, len,
(checksummed ? " [SUMMED]" : ""),
(discard ? " [DISCARD]" : ""));
/* Discard packet, if instructed to do so */
if (unlikely(discard)) {
if (unlikely(leak_packet))
rx_queue->channel->n_skbuff_leaks++;
else
/* We haven't called efx_unmap_rx_buffer yet,
* so fini the entire rx_buffer here */
efx_fini_rx_buffer(rx_queue, rx_buf);
return;
}
/* Release card resources - assumes all RX buffers consumed in-order
* per RX queue
*/
efx_unmap_rx_buffer(efx, rx_buf);
/* Prefetch nice and early so data will (hopefully) be in cache by
* the time we look at it.
*/
prefetch(rx_buf->data);
/* Pipeline receives so that we give time for packet headers to be
* prefetched into cache.
*/
rx_buf->len = len;
if (rx_queue->channel->rx_pkt)
__efx_rx_packet(rx_queue->channel,
rx_queue->channel->rx_pkt,
rx_queue->channel->rx_pkt_csummed);
rx_queue->channel->rx_pkt = rx_buf;
rx_queue->channel->rx_pkt_csummed = checksummed;
}
/* Attempt to recycle the page if there is an RX recycle ring; the page can
* only be added if this is the final RX buffer, to prevent pages being used in
* the descriptor ring and appearing in the recycle ring simultaneously.
*/
static void efx_recycle_rx_page(struct efx_channel *channel,
struct efx_rx_buffer *rx_buf)
{
struct page *page = rx_buf->page;
struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
struct efx_nic *efx = rx_queue->efx;
unsigned index;
/* Only recycle the page after processing the final buffer. */
if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE))
return;
index = rx_queue->page_add & rx_queue->page_ptr_mask;
if (rx_queue->page_ring[index] == NULL) {
unsigned read_index = rx_queue->page_remove &
rx_queue->page_ptr_mask;
/* The next slot in the recycle ring is available, but
* increment page_remove if the read pointer currently
* points here.
*/
if (read_index == index)
++rx_queue->page_remove;
rx_queue->page_ring[index] = page;
++rx_queue->page_add;
return;
}
++rx_queue->page_recycle_full;
efx_unmap_rx_buffer(efx, rx_buf);
put_page(rx_buf->page);
}
static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,
struct efx_rx_buffer *rx_buf)
{
/* Release the page reference we hold for the buffer. */
if (rx_buf->page)
put_page(rx_buf->page);
/* If this is the last buffer in a page, unmap and free it. */
if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) {
efx_unmap_rx_buffer(rx_queue->efx, rx_buf);
efx_free_rx_buffer(rx_buf);
}
rx_buf->page = NULL;
}
static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,
struct efx_rx_buffer *rx_buf)
{
efx_unmap_rx_buffer(rx_queue->efx, rx_buf);
efx_free_rx_buffer(rx_queue->efx, rx_buf);
}