/**
* efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
* @tx_queue: Efx TX queue
* @skb: Socket buffer
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK or %NETDEV_TX_BUSY.
*/
static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
int frag_i, rc, rc2 = NETDEV_TX_OK;
struct tso_state state;
/* Find the packet protocol and sanity-check it */
state.protocol = efx_tso_check_protocol(skb);
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
tso_start(&state, skb);
/* Assume that skb header area contains exactly the headers, and
* all payload is in the frag list.
*/
if (skb_headlen(skb) == state.header_len) {
/* Grab the first payload fragment. */
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
} else {
rc = tso_get_head_fragment(&state, efx, skb);
if (rc)
goto mem_err;
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
while (1) {
rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
if (unlikely(rc)) {
rc2 = NETDEV_TX_BUSY;
goto unwind;
}
/* Move onto the next fragment? */
if (state.in_len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
}
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
tx_queue->tso_bursts++;
return NETDEV_TX_OK;
mem_err:
netif_err(efx, tx_err, efx->net_dev,
"Out of memory for TSO headers, or PCI mapping error\n");
dev_kfree_skb_any(skb);
unwind:
/* Free the DMA mapping we were in the process of writing out */
if (state.unmap_len) {
if (state.unmap_single)
pci_unmap_single(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
else
pci_unmap_page(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
}
efx_enqueue_unwind(tx_queue);
return rc2;
}
/**
* efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
* @tx_queue: Efx TX queue
* @skb: Socket buffer
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK or %NETDEV_TX_BUSY.
*/
static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
const struct sk_buff *skb)
{
int frag_i, rc, rc2 = NETDEV_TX_OK;
struct tso_state state;
skb_frag_t *f;
/* Verify TSO is safe - these checks should never fail. */
efx_tso_check_safe(skb);
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
tso_start(&state, skb);
/* Assume that skb header area contains exactly the headers, and
* all payload is in the frag list.
*/
if (skb_headlen(skb) == state.p.header_length) {
/* Grab the first payload fragment. */
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
f = &skb_shinfo(skb)->frags[frag_i];
rc = tso_get_fragment(&state, tx_queue->efx,
f->size, f->page, f->page_offset);
if (rc)
goto mem_err;
} else {
/* It may look like this code fragment assumes that the
* skb->data portion does not cross a page boundary, but
* that is not the case. It is guaranteed to be direct
* mapped memory, and therefore is physically contiguous,
* and so DMA will work fine. kmap_atomic() on this region
* will just return the direct mapping, so that will work
* too.
*/
int page_off = (unsigned long)skb->data & (PAGE_SIZE - 1);
int hl = state.p.header_length;
rc = tso_get_fragment(&state, tx_queue->efx,
skb_headlen(skb) - hl,
virt_to_page(skb->data), page_off + hl);
if (rc)
goto mem_err;
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
while (1) {
rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
if (unlikely(rc))
goto stop;
/* Move onto the next fragment? */
if (state.ifc.len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
f = &skb_shinfo(skb)->frags[frag_i];
rc = tso_get_fragment(&state, tx_queue->efx,
f->size, f->page, f->page_offset);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
}
/* Pass off to hardware */
falcon_push_buffers(tx_queue);
tx_queue->tso_bursts++;
return NETDEV_TX_OK;
mem_err:
EFX_ERR(tx_queue->efx, "Out of memory for TSO headers, or PCI mapping"
" error\n");
dev_kfree_skb_any((struct sk_buff *)skb);
goto unwind;
stop:
rc2 = NETDEV_TX_BUSY;
/* Stop the queue if it wasn't stopped before. */
if (tx_queue->stopped == 1)
efx_stop_queue(tx_queue->efx);
unwind:
efx_enqueue_unwind(tx_queue);
return rc2;
}
/**
* efx_tx_tso_sw - segment and transmit a TSO socket buffer using SW or FATSOv1
* @tx_queue: Efx TX queue
* @skb: Socket buffer
* @data_mapped: Did we map the data? Always set to true
* by this on success.
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK.
*/
int efx_tx_tso_sw(struct efx_tx_queue *tx_queue, struct sk_buff *skb,
bool *data_mapped)
{
struct efx_nic *efx = tx_queue->efx;
int frag_i, rc;
struct tso_state state;
#if defined(EFX_USE_KCOMPAT) && !defined(EFX_HAVE_GSO_MAX_SEGS)
/* Since the stack does not limit the number of segments per
* skb, we must do so. Otherwise an attacker may be able to
* make the TCP produce skbs that will never fit in our TX
* queue, causing repeated resets.
*/
if (unlikely(skb_shinfo(skb)->gso_segs > EFX_TSO_MAX_SEGS)) {
unsigned int excess =
(skb_shinfo(skb)->gso_segs - EFX_TSO_MAX_SEGS) *
skb_shinfo(skb)->gso_size;
if (__pskb_trim(skb, skb->len - excess))
return -E2BIG;
}
#endif
prefetch(skb->data);
/* Find the packet protocol and sanity-check it */
rc = efx_tso_check_protocol(skb, &state.protocol);
if (rc)
return rc;
rc = tso_start(&state, efx, tx_queue, skb);
if (rc)
goto mem_err;
if (likely(state.in_len == 0)) {
/* Grab the first payload fragment. */
EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
} else {
/* Payload starts in the header area. */
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state, true) < 0)
goto mem_err;
prefetch_ptr(tx_queue);
while (1) {
tso_fill_packet_with_fragment(tx_queue, skb, &state);
/* Move onto the next fragment? */
if (state.in_len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state, false) < 0)
goto mem_err;
}
*data_mapped = true;
return 0;
mem_err:
netif_err(efx, tx_err, efx->net_dev,
"Out of memory for TSO headers, or DMA mapping error\n");
/* Free the DMA mapping we were in the process of writing out */
if (state.unmap_len) {
if (state.dma_flags & EFX_TX_BUF_MAP_SINGLE)
dma_unmap_single(&efx->pci_dev->dev, state.unmap_addr,
state.unmap_len, DMA_TO_DEVICE);
else
dma_unmap_page(&efx->pci_dev->dev, state.unmap_addr,
state.unmap_len, DMA_TO_DEVICE);
}
/* Free the header DMA mapping, if using option descriptors */
if (state.header_unmap_len)
dma_unmap_single(&efx->pci_dev->dev, state.header_dma_addr,
state.header_unmap_len, DMA_TO_DEVICE);
return -ENOMEM;
}
/**
* efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
* @tx_queue: Efx TX queue
* @skb: Socket buffer
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK or %NETDEV_TX_BUSY.
*/
static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
int frag_i, rc, rc2 = NETDEV_TX_OK;
struct tso_state state;
/* Since the stack does not limit the number of segments per
* skb, we must do so. Otherwise an attacker may be able to
* make the TCP produce skbs that will never fit in our TX
* queue, causing repeated resets.
*/
if (unlikely(skb_shinfo(skb)->gso_segs > EFX_TSO_MAX_SEGS)) {
unsigned int excess =
(skb_shinfo(skb)->gso_segs - EFX_TSO_MAX_SEGS) *
skb_shinfo(skb)->gso_size;
if (__pskb_trim(skb, skb->len - excess)) {
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
}
/* Find the packet protocol and sanity-check it */
state.protocol = efx_tso_check_protocol(skb);
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
tso_start(&state, skb);
/* Assume that skb header area contains exactly the headers, and
* all payload is in the frag list.
*/
if (skb_headlen(skb) == state.header_len) {
/* Grab the first payload fragment. */
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
} else {
rc = tso_get_head_fragment(&state, efx, skb);
if (rc)
goto mem_err;
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
while (1) {
rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
if (unlikely(rc)) {
rc2 = NETDEV_TX_BUSY;
goto unwind;
}
/* Move onto the next fragment? */
if (state.in_len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
}
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
tx_queue->tso_bursts++;
return NETDEV_TX_OK;
mem_err:
netif_err(efx, tx_err, efx->net_dev,
"Out of memory for TSO headers, or PCI mapping error\n");
dev_kfree_skb_any(skb);
unwind:
/* Free the DMA mapping we were in the process of writing out */
if (state.unmap_len) {
if (state.unmap_single)
pci_unmap_single(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
else
pci_unmap_page(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
}
efx_enqueue_unwind(tx_queue);
return rc2;
}