/* Remove descriptors put into a tx_queue. */
static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
{
struct efx_tx_buffer *buffer;
dma_addr_t unmap_addr;
/* Work backwards until we hit the original insert pointer value */
while (tx_queue->insert_count != tx_queue->write_count) {
--tx_queue->insert_count;
buffer = &tx_queue->buffer[tx_queue->insert_count &
tx_queue->ptr_mask];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->skb);
if (buffer->unmap_len) {
unmap_addr = (buffer->dma_addr + buffer->len -
buffer->unmap_len);
if (buffer->unmap_single)
pci_unmap_single(tx_queue->efx->pci_dev,
unmap_addr, buffer->unmap_len,
PCI_DMA_TODEVICE);
else
pci_unmap_page(tx_queue->efx->pci_dev,
unmap_addr, buffer->unmap_len,
PCI_DMA_TODEVICE);
buffer->unmap_len = 0;
}
buffer->len = 0;
buffer->continuation = true;
}
}
/*
* Free up all TSO datastructures associated with tx_queue. This
* routine should be called only once the tx_queue is both empty and
* will no longer be used.
*/
static void efx_fini_tso(struct efx_tx_queue *tx_queue)
{
unsigned i;
if (tx_queue->buffer) {
for (i = 0; i <= tx_queue->ptr_mask; ++i)
efx_tsoh_free(tx_queue, &tx_queue->buffer[i]);
}
while (tx_queue->tso_headers_free != NULL)
efx_tsoh_block_free(tx_queue, tx_queue->tso_headers_free,
tx_queue->efx->pci_dev);
}
/*
* Put a TSO header into the TX queue.
*
* This is special-cased because we know that it is small enough to fit in
* a single fragment, and we know it doesn't cross a page boundary. It
* also allows us to not worry about end-of-packet etc.
*/
static void efx_tso_put_header(struct efx_tx_queue *tx_queue,
struct efx_tso_header *tsoh, unsigned len)
{
struct efx_tx_buffer *buffer;
buffer = &tx_queue->buffer[tx_queue->insert_count & tx_queue->ptr_mask];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(!buffer->continuation);
EFX_BUG_ON_PARANOID(buffer->tsoh);
buffer->len = len;
buffer->dma_addr = tsoh->dma_addr;
buffer->tsoh = tsoh;
++tx_queue->insert_count;
}
/**
* efx_tx_queue_insert - push descriptors onto the TX queue
* @tx_queue: Efx TX queue
* @dma_addr: DMA address of fragment
* @len: Length of fragment
* @final_buffer: The final buffer inserted into the queue
*
* Push descriptors onto the TX queue. Return 0 on success or 1 if
* @tx_queue full.
*/
static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
dma_addr_t dma_addr, unsigned len,
struct efx_tx_buffer **final_buffer)
{
struct efx_tx_buffer *buffer;
struct efx_nic *efx = tx_queue->efx;
unsigned dma_len, fill_level, insert_ptr;
int q_space;
EFX_BUG_ON_PARANOID(len <= 0);
fill_level = tx_queue->insert_count - tx_queue->old_read_count;
/* -1 as there is no way to represent all descriptors used */
q_space = efx->txq_entries - 1 - fill_level;
while (1) {
if (unlikely(q_space-- <= 0)) {
/* It might be that completions have happened
* since the xmit path last checked. Update
* the xmit path's copy of read_count.
*/
netif_tx_stop_queue(tx_queue->core_txq);
/* This memory barrier protects the change of
* queue state from the access of read_count. */
smp_mb();
tx_queue->old_read_count =
ACCESS_ONCE(tx_queue->read_count);
fill_level = (tx_queue->insert_count
- tx_queue->old_read_count);
q_space = efx->txq_entries - 1 - fill_level;
if (unlikely(q_space-- <= 0)) {
*final_buffer = NULL;
return 1;
}
smp_mb();
netif_tx_start_queue(tx_queue->core_txq);
}
insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
buffer = &tx_queue->buffer[insert_ptr];
++tx_queue->insert_count;
EFX_BUG_ON_PARANOID(tx_queue->insert_count -
tx_queue->read_count >=
efx->txq_entries);
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(!buffer->continuation);
EFX_BUG_ON_PARANOID(buffer->tsoh);
buffer->dma_addr = dma_addr;
dma_len = efx_max_tx_len(efx, dma_addr);
/* If there is enough space to send then do so */
if (dma_len >= len)
break;
buffer->len = dma_len; /* Don't set the other members */
dma_addr += dma_len;
len -= dma_len;
}
EFX_BUG_ON_PARANOID(!len);
buffer->len = len;
*final_buffer = buffer;
return 0;
}
/*
* Add a socket buffer to a TX queue
*
* This maps all fragments of a socket buffer for DMA and adds them to
* the TX queue. The queue's insert pointer will be incremented by
* the number of fragments in the socket buffer.
*
* If any DMA mapping fails, any mapped fragments will be unmapped,
* the queue's insert pointer will be restored to its original value.
*
* This function is split out from efx_hard_start_xmit to allow the
* loopback test to direct packets via specific TX queues.
*
* Returns NETDEV_TX_OK or NETDEV_TX_BUSY
* You must hold netif_tx_lock() to call this function.
*/
netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
struct pci_dev *pci_dev = efx->pci_dev;
struct efx_tx_buffer *buffer;
skb_frag_t *fragment;
struct page *page;
int page_offset;
unsigned int len, unmap_len = 0, fill_level, insert_ptr;
dma_addr_t dma_addr, unmap_addr = 0;
unsigned int dma_len;
bool unmap_single;
int q_space, i = 0;
netdev_tx_t rc = NETDEV_TX_OK;
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
if (skb_shinfo(skb)->gso_size)
return efx_enqueue_skb_tso(tx_queue, skb);
/* Get size of the initial fragment */
len = skb_headlen(skb);
/* Pad if necessary */
if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) {
EFX_BUG_ON_PARANOID(skb->data_len);
len = 32 + 1;
if (skb_pad(skb, len - skb->len))
return NETDEV_TX_OK;
}
fill_level = tx_queue->insert_count - tx_queue->old_read_count;
q_space = efx->txq_entries - 1 - fill_level;
/* Map for DMA. Use pci_map_single rather than pci_map_page
* since this is more efficient on machines with sparse
* memory.
*/
unmap_single = true;
dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
/* Process all fragments */
while (1) {
if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
goto pci_err;
/* Store fields for marking in the per-fragment final
* descriptor */
unmap_len = len;
unmap_addr = dma_addr;
/* Add to TX queue, splitting across DMA boundaries */
do {
if (unlikely(q_space-- <= 0)) {
/* It might be that completions have
* happened since the xmit path last
* checked. Update the xmit path's
* copy of read_count.
*/
netif_tx_stop_queue(tx_queue->core_txq);
/* This memory barrier protects the
* change of queue state from the access
* of read_count. */
smp_mb();
tx_queue->old_read_count =
ACCESS_ONCE(tx_queue->read_count);
fill_level = (tx_queue->insert_count
- tx_queue->old_read_count);
q_space = efx->txq_entries - 1 - fill_level;
if (unlikely(q_space-- <= 0)) {
rc = NETDEV_TX_BUSY;
goto unwind;
}
smp_mb();
netif_tx_start_queue(tx_queue->core_txq);
}
insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
buffer = &tx_queue->buffer[insert_ptr];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->tsoh);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(!buffer->continuation);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
dma_len = efx_max_tx_len(efx, dma_addr);
if (likely(dma_len >= len))
dma_len = len;
/* Fill out per descriptor fields */
buffer->len = dma_len;
buffer->dma_addr = dma_addr;
len -= dma_len;
dma_addr += dma_len;
++tx_queue->insert_count;
} while (len);
/* Transfer ownership of the unmapping to the final buffer */
buffer->unmap_single = unmap_single;
buffer->unmap_len = unmap_len;
unmap_len = 0;
/* Get address and size of next fragment */
if (i >= skb_shinfo(skb)->nr_frags)
break;
fragment = &skb_shinfo(skb)->frags[i];
len = fragment->size;
page = fragment->page;
page_offset = fragment->page_offset;
i++;
/* Map for DMA */
unmap_single = false;
dma_addr = pci_map_page(pci_dev, page, page_offset, len,
PCI_DMA_TODEVICE);
}
/* Transfer ownership of the skb to the final buffer */
buffer->skb = skb;
buffer->continuation = false;
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
return NETDEV_TX_OK;
pci_err:
netif_err(efx, tx_err, efx->net_dev,
" TX queue %d could not map skb with %d bytes %d "
"fragments for DMA\n", tx_queue->queue, skb->len,
skb_shinfo(skb)->nr_frags + 1);
/* Mark the packet as transmitted, and free the SKB ourselves */
dev_kfree_skb_any(skb);
unwind:
/* Work backwards until we hit the original insert pointer value */
while (tx_queue->insert_count != tx_queue->write_count) {
--tx_queue->insert_count;
insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
buffer = &tx_queue->buffer[insert_ptr];
efx_dequeue_buffer(tx_queue, buffer);
buffer->len = 0;
}
/* Free the fragment we were mid-way through pushing */
if (unmap_len) {
if (unmap_single)
pci_unmap_single(pci_dev, unmap_addr, unmap_len,
PCI_DMA_TODEVICE);
else
pci_unmap_page(pci_dev, unmap_addr, unmap_len,
PCI_DMA_TODEVICE);
}
return rc;
}
/**
* efx_tx_queue_insert - push descriptors onto the TX queue
* @tx_queue: Efx TX queue
* @dma_addr: DMA address of fragment
* @len: Length of fragment
* @final_buffer: The final buffer inserted into the queue
*
* Push descriptors onto the TX queue. Return 0 on success or 1 if
* @tx_queue full.
*/
static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
dma_addr_t dma_addr, unsigned len,
struct efx_tx_buffer **final_buffer)
{
struct efx_tx_buffer *buffer;
struct efx_nic *efx = tx_queue->efx;
unsigned dma_len, fill_level, insert_ptr, misalign;
int q_space;
EFX_BUG_ON_PARANOID(len <= 0);
fill_level = tx_queue->insert_count - tx_queue->old_read_count;
/* -1 as there is no way to represent all descriptors used */
q_space = efx->type->txd_ring_mask - 1 - fill_level;
while (1) {
if (unlikely(q_space-- <= 0)) {
/* It might be that completions have happened
* since the xmit path last checked. Update
* the xmit path's copy of read_count.
*/
++tx_queue->stopped;
/* This memory barrier protects the change of
* stopped from the access of read_count. */
smp_mb();
tx_queue->old_read_count =
*(volatile unsigned *)&tx_queue->read_count;
fill_level = (tx_queue->insert_count
- tx_queue->old_read_count);
q_space = efx->type->txd_ring_mask - 1 - fill_level;
if (unlikely(q_space-- <= 0)) {
*final_buffer = NULL;
return 1;
}
smp_mb();
--tx_queue->stopped;
}
insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask;
buffer = &tx_queue->buffer[insert_ptr];
++tx_queue->insert_count;
EFX_BUG_ON_PARANOID(tx_queue->insert_count -
tx_queue->read_count >
efx->type->txd_ring_mask);
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(!buffer->continuation);
EFX_BUG_ON_PARANOID(buffer->tsoh);
buffer->dma_addr = dma_addr;
/* Ensure we do not cross a boundary unsupported by H/W */
dma_len = (~dma_addr & efx->type->tx_dma_mask) + 1;
misalign = (unsigned)dma_addr & efx->type->bug5391_mask;
if (misalign && dma_len + misalign > 512)
dma_len = 512 - misalign;
/* If there is enough space to send then do so */
if (dma_len >= len)
break;
buffer->len = dma_len; /* Don't set the other members */
dma_addr += dma_len;
len -= dma_len;
}
EFX_BUG_ON_PARANOID(!len);
buffer->len = len;
*final_buffer = buffer;
return 0;
}
/*
* Add a socket buffer to a TX queue
*
* This maps all fragments of a socket buffer for DMA and adds them to
* the TX queue. The queue's insert pointer will be incremented by
* the number of fragments in the socket buffer.
*
* If any DMA mapping fails, any mapped fragments will be unmapped,
* the queue's insert pointer will be restored to its original value.
*
* Returns NETDEV_TX_OK or NETDEV_TX_BUSY
* You must hold netif_tx_lock() to call this function.
*/
static inline int efx_enqueue_skb(struct efx_tx_queue *tx_queue,
const struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
struct pci_dev *pci_dev = efx->pci_dev;
struct efx_tx_buffer *buffer;
skb_frag_t *fragment;
struct page *page;
int page_offset;
unsigned int len, unmap_len = 0, fill_level, insert_ptr, misalign;
dma_addr_t dma_addr, unmap_addr = 0;
unsigned int dma_len;
unsigned unmap_single;
int q_space, i = 0;
int rc = NETDEV_TX_OK;
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
if (skb_shinfo((struct sk_buff *)skb)->gso_size)
return efx_enqueue_skb_tso(tx_queue, skb);
/* Get size of the initial fragment */
len = skb_headlen(skb);
fill_level = tx_queue->insert_count - tx_queue->old_read_count;
q_space = efx->type->txd_ring_mask - 1 - fill_level;
/* Map for DMA. Use pci_map_single rather than pci_map_page
* since this is more efficient on machines with sparse
* memory.
*/
unmap_single = 1;
dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
/* Process all fragments */
while (1) {
if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
goto pci_err;
/* Store fields for marking in the per-fragment final
* descriptor */
unmap_len = len;
unmap_addr = dma_addr;
/* Add to TX queue, splitting across DMA boundaries */
do {
if (unlikely(q_space-- <= 0)) {
/* It might be that completions have
* happened since the xmit path last
* checked. Update the xmit path's
* copy of read_count.
*/
++tx_queue->stopped;
/* This memory barrier protects the
* change of stopped from the access
* of read_count. */
smp_mb();
tx_queue->old_read_count =
*(volatile unsigned *)
&tx_queue->read_count;
fill_level = (tx_queue->insert_count
- tx_queue->old_read_count);
q_space = (efx->type->txd_ring_mask - 1 -
fill_level);
if (unlikely(q_space-- <= 0))
goto stop;
smp_mb();
--tx_queue->stopped;
}
insert_ptr = (tx_queue->insert_count &
efx->type->txd_ring_mask);
buffer = &tx_queue->buffer[insert_ptr];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->tsoh);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(buffer->continuation != 1);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
dma_len = (((~dma_addr) & efx->type->tx_dma_mask) + 1);
if (likely(dma_len > len))
dma_len = len;
misalign = (unsigned)dma_addr & efx->type->bug5391_mask;
if (misalign && dma_len + misalign > 512)
dma_len = 512 - misalign;
/* Fill out per descriptor fields */
buffer->len = dma_len;
buffer->dma_addr = dma_addr;
len -= dma_len;
dma_addr += dma_len;
++tx_queue->insert_count;
} while (len);
/* Transfer ownership of the unmapping to the final buffer */
buffer->unmap_addr = unmap_addr;
buffer->unmap_single = unmap_single;
buffer->unmap_len = unmap_len;
unmap_len = 0;
/* Get address and size of next fragment */
if (i >= skb_shinfo(skb)->nr_frags)
break;
fragment = &skb_shinfo(skb)->frags[i];
len = fragment->size;
page = fragment->page;
page_offset = fragment->page_offset;
i++;
/* Map for DMA */
unmap_single = 0;
dma_addr = pci_map_page(pci_dev, page, page_offset, len,
PCI_DMA_TODEVICE);
}
/* Transfer ownership of the skb to the final buffer */
buffer->skb = skb;
buffer->continuation = 0;
/* Pass off to hardware */
falcon_push_buffers(tx_queue);
return NETDEV_TX_OK;
pci_err:
EFX_ERR_RL(efx, " TX queue %d could not map skb with %d bytes %d "
"fragments for DMA\n", tx_queue->queue, skb->len,
skb_shinfo(skb)->nr_frags + 1);
/* Mark the packet as transmitted, and free the SKB ourselves */
dev_kfree_skb_any((struct sk_buff *)skb);
goto unwind;
stop:
rc = NETDEV_TX_BUSY;
if (tx_queue->stopped == 1)
efx_stop_queue(efx);
unwind:
/* Work backwards until we hit the original insert pointer value */
while (tx_queue->insert_count != tx_queue->write_count) {
--tx_queue->insert_count;
insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask;
buffer = &tx_queue->buffer[insert_ptr];
efx_dequeue_buffer(tx_queue, buffer);
buffer->len = 0;
}
/* Free the fragment we were mid-way through pushing */
if (unmap_len)
pci_unmap_page(pci_dev, unmap_addr, unmap_len,
PCI_DMA_TODEVICE);
return rc;
}
