/**
* efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
* @tx_queue: Efx TX queue
* @skb: Socket buffer
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK or %NETDEV_TX_BUSY.
*/
static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
int frag_i, rc, rc2 = NETDEV_TX_OK;
struct tso_state state;
/* Find the packet protocol and sanity-check it */
state.protocol = efx_tso_check_protocol(skb);
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
tso_start(&state, skb);
/* Assume that skb header area contains exactly the headers, and
* all payload is in the frag list.
*/
if (skb_headlen(skb) == state.header_len) {
/* Grab the first payload fragment. */
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
} else {
rc = tso_get_head_fragment(&state, efx, skb);
if (rc)
goto mem_err;
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
while (1) {
rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
if (unlikely(rc)) {
rc2 = NETDEV_TX_BUSY;
goto unwind;
}
/* Move onto the next fragment? */
if (state.in_len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
}
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
tx_queue->tso_bursts++;
return NETDEV_TX_OK;
mem_err:
netif_err(efx, tx_err, efx->net_dev,
"Out of memory for TSO headers, or PCI mapping error\n");
dev_kfree_skb_any(skb);
unwind:
/* Free the DMA mapping we were in the process of writing out */
if (state.unmap_len) {
if (state.unmap_single)
pci_unmap_single(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
else
pci_unmap_page(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
}
efx_enqueue_unwind(tx_queue);
return rc2;
}
/**
* efx_tx_tso_sw - segment and transmit a TSO socket buffer using SW or FATSOv1
* @tx_queue: Efx TX queue
* @skb: Socket buffer
* @data_mapped: Did we map the data? Always set to true
* by this on success.
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK.
*/
int efx_tx_tso_sw(struct efx_tx_queue *tx_queue, struct sk_buff *skb,
bool *data_mapped)
{
struct efx_nic *efx = tx_queue->efx;
int frag_i, rc;
struct tso_state state;
#if defined(EFX_USE_KCOMPAT) && !defined(EFX_HAVE_GSO_MAX_SEGS)
/* Since the stack does not limit the number of segments per
* skb, we must do so. Otherwise an attacker may be able to
* make the TCP produce skbs that will never fit in our TX
* queue, causing repeated resets.
*/
if (unlikely(skb_shinfo(skb)->gso_segs > EFX_TSO_MAX_SEGS)) {
unsigned int excess =
(skb_shinfo(skb)->gso_segs - EFX_TSO_MAX_SEGS) *
skb_shinfo(skb)->gso_size;
if (__pskb_trim(skb, skb->len - excess))
return -E2BIG;
}
#endif
prefetch(skb->data);
/* Find the packet protocol and sanity-check it */
rc = efx_tso_check_protocol(skb, &state.protocol);
if (rc)
return rc;
rc = tso_start(&state, efx, tx_queue, skb);
if (rc)
goto mem_err;
if (likely(state.in_len == 0)) {
/* Grab the first payload fragment. */
EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
} else {
/* Payload starts in the header area. */
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state, true) < 0)
goto mem_err;
prefetch_ptr(tx_queue);
while (1) {
tso_fill_packet_with_fragment(tx_queue, skb, &state);
/* Move onto the next fragment? */
if (state.in_len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state, false) < 0)
goto mem_err;
}
*data_mapped = true;
return 0;
mem_err:
netif_err(efx, tx_err, efx->net_dev,
"Out of memory for TSO headers, or DMA mapping error\n");
/* Free the DMA mapping we were in the process of writing out */
if (state.unmap_len) {
if (state.dma_flags & EFX_TX_BUF_MAP_SINGLE)
dma_unmap_single(&efx->pci_dev->dev, state.unmap_addr,
state.unmap_len, DMA_TO_DEVICE);
else
dma_unmap_page(&efx->pci_dev->dev, state.unmap_addr,
state.unmap_len, DMA_TO_DEVICE);
}
/* Free the header DMA mapping, if using option descriptors */
if (state.header_unmap_len)
dma_unmap_single(&efx->pci_dev->dev, state.header_dma_addr,
state.header_unmap_len, DMA_TO_DEVICE);
return -ENOMEM;
}
static int iwl_pcie_gen2_build_amsdu(struct iwl_trans *trans,
struct sk_buff *skb,
struct iwl_tfh_tfd *tfd, int start_len,
u8 hdr_len, struct iwl_device_cmd *dev_cmd)
{
#ifdef CONFIG_INET
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
struct iwl_tx_cmd_gen2 *tx_cmd = (void *)dev_cmd->payload;
struct ieee80211_hdr *hdr = (void *)skb->data;
unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room;
unsigned int mss = skb_shinfo(skb)->gso_size;
u16 length, iv_len, amsdu_pad;
u8 *start_hdr;
struct iwl_tso_hdr_page *hdr_page;
struct page **page_ptr;
struct tso_t tso;
/* if the packet is protected, then it must be CCMP or GCMP */
iv_len = ieee80211_has_protected(hdr->frame_control) ?
IEEE80211_CCMP_HDR_LEN : 0;
trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd),
&dev_cmd->hdr, start_len, 0);
ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb);
snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb);
total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len;
amsdu_pad = 0;
/* total amount of header we may need for this A-MSDU */
hdr_room = DIV_ROUND_UP(total_len, mss) *
(3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len;
/* Our device supports 9 segments at most, it will fit in 1 page */
hdr_page = get_page_hdr(trans, hdr_room);
if (!hdr_page)
return -ENOMEM;
get_page(hdr_page->page);
start_hdr = hdr_page->pos;
page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs);
*page_ptr = hdr_page->page;
memcpy(hdr_page->pos, skb->data + hdr_len, iv_len);
hdr_page->pos += iv_len;
/*
* Pull the ieee80211 header + IV to be able to use TSO core,
* we will restore it for the tx_status flow.
*/
skb_pull(skb, hdr_len + iv_len);
/*
* Remove the length of all the headers that we don't actually
* have in the MPDU by themselves, but that we duplicate into
* all the different MSDUs inside the A-MSDU.
*/
le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen);
tso_start(skb, &tso);
while (total_len) {
/* this is the data left for this subframe */
unsigned int data_left = min_t(unsigned int, mss, total_len);
struct sk_buff *csum_skb = NULL;
unsigned int tb_len;
dma_addr_t tb_phys;
u8 *subf_hdrs_start = hdr_page->pos;
total_len -= data_left;
memset(hdr_page->pos, 0, amsdu_pad);
hdr_page->pos += amsdu_pad;
amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen +
data_left)) & 0x3;
ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr));
hdr_page->pos += ETH_ALEN;
ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr));
hdr_page->pos += ETH_ALEN;
length = snap_ip_tcp_hdrlen + data_left;
*((__be16 *)hdr_page->pos) = cpu_to_be16(length);
hdr_page->pos += sizeof(length);
/*
* This will copy the SNAP as well which will be considered
* as MAC header.
*/
tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len);
hdr_page->pos += snap_ip_tcp_hdrlen;
tb_len = hdr_page->pos - start_hdr;
tb_phys = dma_map_single(trans->dev, start_hdr,
tb_len, DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(trans->dev, tb_phys))) {
dev_kfree_skb(csum_skb);
goto out_err;
}
iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb_len);
trace_iwlwifi_dev_tx_tb(trans->dev, skb, start_hdr, tb_len);
/* add this subframe's headers' length to the tx_cmd */
le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start);
/* prepare the start_hdr for the next subframe */
start_hdr = hdr_page->pos;
/* put the payload */
while (data_left) {
tb_len = min_t(unsigned int, tso.size, data_left);
tb_phys = dma_map_single(trans->dev, tso.data,
tb_len, DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(trans->dev, tb_phys))) {
dev_kfree_skb(csum_skb);
goto out_err;
}
iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb_len);
trace_iwlwifi_dev_tx_tb(trans->dev, skb, tso.data,
tb_len);
data_left -= tb_len;
tso_build_data(skb, &tso, tb_len);
}
}
/* re -add the WiFi header and IV */
skb_push(skb, hdr_len + iv_len);
return 0;
out_err:
#endif
return -EINVAL;
}
/**
* efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
* @tx_queue: Efx TX queue
* @skb: Socket buffer
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK or %NETDEV_TX_BUSY.
*/
static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
const struct sk_buff *skb)
{
int frag_i, rc, rc2 = NETDEV_TX_OK;
struct tso_state state;
skb_frag_t *f;
/* Verify TSO is safe - these checks should never fail. */
efx_tso_check_safe(skb);
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
tso_start(&state, skb);
/* Assume that skb header area contains exactly the headers, and
* all payload is in the frag list.
*/
if (skb_headlen(skb) == state.p.header_length) {
/* Grab the first payload fragment. */
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
f = &skb_shinfo(skb)->frags[frag_i];
rc = tso_get_fragment(&state, tx_queue->efx,
f->size, f->page, f->page_offset);
if (rc)
goto mem_err;
} else {
/* It may look like this code fragment assumes that the
* skb->data portion does not cross a page boundary, but
* that is not the case. It is guaranteed to be direct
* mapped memory, and therefore is physically contiguous,
* and so DMA will work fine. kmap_atomic() on this region
* will just return the direct mapping, so that will work
* too.
*/
int page_off = (unsigned long)skb->data & (PAGE_SIZE - 1);
int hl = state.p.header_length;
rc = tso_get_fragment(&state, tx_queue->efx,
skb_headlen(skb) - hl,
virt_to_page(skb->data), page_off + hl);
if (rc)
goto mem_err;
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
while (1) {
rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
if (unlikely(rc))
goto stop;
/* Move onto the next fragment? */
if (state.ifc.len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
f = &skb_shinfo(skb)->frags[frag_i];
rc = tso_get_fragment(&state, tx_queue->efx,
f->size, f->page, f->page_offset);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
}
/* Pass off to hardware */
falcon_push_buffers(tx_queue);
tx_queue->tso_bursts++;
return NETDEV_TX_OK;
mem_err:
EFX_ERR(tx_queue->efx, "Out of memory for TSO headers, or PCI mapping"
" error\n");
dev_kfree_skb_any((struct sk_buff *)skb);
goto unwind;
stop:
rc2 = NETDEV_TX_BUSY;
/* Stop the queue if it wasn't stopped before. */
if (tx_queue->stopped == 1)
efx_stop_queue(tx_queue->efx);
unwind:
efx_enqueue_unwind(tx_queue);
return rc2;
}
/**
* efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
* @tx_queue: Efx TX queue
* @skb: Socket buffer
*
* Context: You must hold netif_tx_lock() to call this function.
*
* Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
* @skb was not enqueued. In all cases @skb is consumed. Return
* %NETDEV_TX_OK or %NETDEV_TX_BUSY.
*/
static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
int frag_i, rc, rc2 = NETDEV_TX_OK;
struct tso_state state;
/* Since the stack does not limit the number of segments per
* skb, we must do so. Otherwise an attacker may be able to
* make the TCP produce skbs that will never fit in our TX
* queue, causing repeated resets.
*/
if (unlikely(skb_shinfo(skb)->gso_segs > EFX_TSO_MAX_SEGS)) {
unsigned int excess =
(skb_shinfo(skb)->gso_segs - EFX_TSO_MAX_SEGS) *
skb_shinfo(skb)->gso_size;
if (__pskb_trim(skb, skb->len - excess)) {
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
}
/* Find the packet protocol and sanity-check it */
state.protocol = efx_tso_check_protocol(skb);
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
tso_start(&state, skb);
/* Assume that skb header area contains exactly the headers, and
* all payload is in the frag list.
*/
if (skb_headlen(skb) == state.header_len) {
/* Grab the first payload fragment. */
EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
frag_i = 0;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
} else {
rc = tso_get_head_fragment(&state, efx, skb);
if (rc)
goto mem_err;
frag_i = -1;
}
if (tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
while (1) {
rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
if (unlikely(rc)) {
rc2 = NETDEV_TX_BUSY;
goto unwind;
}
/* Move onto the next fragment? */
if (state.in_len == 0) {
if (++frag_i >= skb_shinfo(skb)->nr_frags)
/* End of payload reached. */
break;
rc = tso_get_fragment(&state, efx,
skb_shinfo(skb)->frags + frag_i);
if (rc)
goto mem_err;
}
/* Start at new packet? */
if (state.packet_space == 0 &&
tso_start_new_packet(tx_queue, skb, &state) < 0)
goto mem_err;
}
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
tx_queue->tso_bursts++;
return NETDEV_TX_OK;
mem_err:
netif_err(efx, tx_err, efx->net_dev,
"Out of memory for TSO headers, or PCI mapping error\n");
dev_kfree_skb_any(skb);
unwind:
/* Free the DMA mapping we were in the process of writing out */
if (state.unmap_len) {
if (state.unmap_single)
pci_unmap_single(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
else
pci_unmap_page(efx->pci_dev, state.unmap_addr,
state.unmap_len, PCI_DMA_TODEVICE);
}
efx_enqueue_unwind(tx_queue);
return rc2;
}
