static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf, int len, bool *discard, bool *leak_packet) { struct efx_nic *efx = rx_queue->efx; unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding; if (likely(len <= max_len)) return; /* The packet must be discarded, but this is only a fatal error * if the caller indicated it was */ *discard = true; if ((len > rx_buf->len) && EFX_WORKAROUND_8071(efx)) { EFX_ERR_RL(efx, " RX queue %d seriously overlength " "RX event (0x%x > 0x%x+0x%x). Leaking\n", rx_queue->queue, len, max_len, efx->type->rx_buffer_padding); /* If this buffer was skb-allocated, then the meta * data at the end of the skb will be trashed. So * we have no choice but to leak the fragment. */ *leak_packet = (rx_buf->skb != NULL); efx_schedule_reset(efx, RESET_TYPE_RX_RECOVERY); } else { EFX_ERR_RL(efx, " RX queue %d overlength RX event " "(0x%x > 0x%x)\n", rx_queue->queue, len, max_len); } rx_queue->channel->n_rx_overlength++; }
/* Allocate and construct an SKB around a struct page.*/ static struct sk_buff *efx_rx_mk_skb(struct efx_rx_buffer *rx_buf, struct efx_nic *efx, int hdr_len) { struct sk_buff *skb; /* Allocate an SKB to store the headers */ skb = netdev_alloc_skb(efx->net_dev, hdr_len + EFX_PAGE_SKB_ALIGN); if (unlikely(skb == NULL)) { EFX_ERR_RL(efx, "RX out of memory for skb\n"); return NULL; } EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags); EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len); skb->ip_summed = CHECKSUM_UNNECESSARY; skb_reserve(skb, EFX_PAGE_SKB_ALIGN); skb->len = rx_buf->len; skb->truesize = rx_buf->len + sizeof(struct sk_buff); memcpy(skb->data, rx_buf->data, hdr_len); skb->tail += hdr_len; /* Append the remaining page onto the frag list */ if (unlikely(rx_buf->len > hdr_len)) { struct skb_frag_struct *frag = skb_shinfo(skb)->frags; frag->page = rx_buf->page; frag->page_offset = efx_rx_buf_offset(rx_buf) + hdr_len; frag->size = skb->len - hdr_len; skb_shinfo(skb)->nr_frags = 1; skb->data_len = frag->size; } else { __free_pages(rx_buf->page, efx->rx_buffer_order); skb->data_len = 0; } /* Ownership has transferred from the rx_buf to skb */ rx_buf->page = NULL; /* Move past the ethernet header */ skb->protocol = eth_type_trans(skb, efx->net_dev); return skb; }
/* * Add a socket buffer to a TX queue * * This maps all fragments of a socket buffer for DMA and adds them to * the TX queue. The queue's insert pointer will be incremented by * the number of fragments in the socket buffer. * * If any DMA mapping fails, any mapped fragments will be unmapped, * the queue's insert pointer will be restored to its original value. * * Returns NETDEV_TX_OK or NETDEV_TX_BUSY * You must hold netif_tx_lock() to call this function. */ static netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb) { struct efx_nic *efx = tx_queue->efx; struct pci_dev *pci_dev = efx->pci_dev; struct efx_tx_buffer *buffer; skb_frag_t *fragment; struct page *page; int page_offset; unsigned int len, unmap_len = 0, fill_level, insert_ptr, misalign; dma_addr_t dma_addr, unmap_addr = 0; unsigned int dma_len; bool unmap_single; int q_space, i = 0; netdev_tx_t rc = NETDEV_TX_OK; EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count); if (skb_shinfo((struct sk_buff *)skb)->gso_size) return efx_enqueue_skb_tso(tx_queue, skb); /* Get size of the initial fragment */ len = skb_headlen(skb); /* Pad if necessary */ if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) { EFX_BUG_ON_PARANOID(skb->data_len); len = 32 + 1; if (skb_pad(skb, len - skb->len)) return NETDEV_TX_OK; } fill_level = tx_queue->insert_count - tx_queue->old_read_count; q_space = efx->type->txd_ring_mask - 1 - fill_level; /* Map for DMA. Use pci_map_single rather than pci_map_page * since this is more efficient on machines with sparse * memory. */ unmap_single = true; dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE); /* Process all fragments */ while (1) { if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr))) goto pci_err; /* Store fields for marking in the per-fragment final * descriptor */ unmap_len = len; unmap_addr = dma_addr; /* Add to TX queue, splitting across DMA boundaries */ do { if (unlikely(q_space-- <= 0)) { /* It might be that completions have * happened since the xmit path last * checked. Update the xmit path's * copy of read_count. */ ++tx_queue->stopped; /* This memory barrier protects the * change of stopped from the access * of read_count. */ smp_mb(); tx_queue->old_read_count = *(volatile unsigned *) &tx_queue->read_count; fill_level = (tx_queue->insert_count - tx_queue->old_read_count); q_space = (efx->type->txd_ring_mask - 1 - fill_level); if (unlikely(q_space-- <= 0)) goto stop; smp_mb(); --tx_queue->stopped; } insert_ptr = (tx_queue->insert_count & efx->type->txd_ring_mask); buffer = &tx_queue->buffer[insert_ptr]; efx_tsoh_free(tx_queue, buffer); EFX_BUG_ON_PARANOID(buffer->tsoh); EFX_BUG_ON_PARANOID(buffer->skb); EFX_BUG_ON_PARANOID(buffer->len); EFX_BUG_ON_PARANOID(!buffer->continuation); EFX_BUG_ON_PARANOID(buffer->unmap_len); dma_len = (((~dma_addr) & efx->type->tx_dma_mask) + 1); if (likely(dma_len > len)) dma_len = len; misalign = (unsigned)dma_addr & efx->type->bug5391_mask; if (misalign && dma_len + misalign > 512) dma_len = 512 - misalign; /* Fill out per descriptor fields */ buffer->len = dma_len; buffer->dma_addr = dma_addr; len -= dma_len; dma_addr += dma_len; ++tx_queue->insert_count; } while (len); /* Transfer ownership of the unmapping to the final buffer */ buffer->unmap_single = unmap_single; buffer->unmap_len = unmap_len; unmap_len = 0; /* Get address and size of next fragment */ if (i >= skb_shinfo(skb)->nr_frags) break; fragment = &skb_shinfo(skb)->frags[i]; len = fragment->size; page = fragment->page; page_offset = fragment->page_offset; i++; /* Map for DMA */ unmap_single = false; dma_addr = pci_map_page(pci_dev, page, page_offset, len, PCI_DMA_TODEVICE); } /* Transfer ownership of the skb to the final buffer */ buffer->skb = skb; buffer->continuation = false; /* Pass off to hardware */ falcon_push_buffers(tx_queue); return NETDEV_TX_OK; pci_err: EFX_ERR_RL(efx, " TX queue %d could not map skb with %d bytes %d " "fragments for DMA\n", tx_queue->queue, skb->len, skb_shinfo(skb)->nr_frags + 1); /* Mark the packet as transmitted, and free the SKB ourselves */ dev_kfree_skb_any((struct sk_buff *)skb); goto unwind; stop: rc = NETDEV_TX_BUSY; if (tx_queue->stopped == 1) efx_stop_queue(efx); unwind: /* Work backwards until we hit the original insert pointer value */ while (tx_queue->insert_count != tx_queue->write_count) { --tx_queue->insert_count; insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask; buffer = &tx_queue->buffer[insert_ptr]; efx_dequeue_buffer(tx_queue, buffer); buffer->len = 0; } /* Free the fragment we were mid-way through pushing */ if (unmap_len) { if (unmap_single) pci_unmap_single(pci_dev, unmap_addr, unmap_len, PCI_DMA_TODEVICE); else pci_unmap_page(pci_dev, unmap_addr, unmap_len, PCI_DMA_TODEVICE); } return rc; }