static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget)
{
union ixgbe_adv_tx_desc *tx_desc = NULL;
struct ixgbe_tx_buffer *tx_bi;
bool work_done = true;
u32 len, cmd_type;
dma_addr_t dma;
while (budget-- > 0) {
if (unlikely(!ixgbe_desc_unused(xdp_ring)) ||
!netif_carrier_ok(xdp_ring->netdev)) {
work_done = false;
break;
}
if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len))
break;
dma_sync_single_for_device(xdp_ring->dev, dma, len,
DMA_BIDIRECTIONAL);
tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use];
tx_bi->bytecount = len;
tx_bi->xdpf = NULL;
tx_desc = IXGBE_TX_DESC(xdp_ring, xdp_ring->next_to_use);
tx_desc->read.buffer_addr = cpu_to_le64(dma);
/* put descriptor type bits */
cmd_type = IXGBE_ADVTXD_DTYP_DATA |
IXGBE_ADVTXD_DCMD_DEXT |
IXGBE_ADVTXD_DCMD_IFCS;
cmd_type |= len | IXGBE_TXD_CMD;
tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
tx_desc->read.olinfo_status =
cpu_to_le32(len << IXGBE_ADVTXD_PAYLEN_SHIFT);
xdp_ring->next_to_use++;
if (xdp_ring->next_to_use == xdp_ring->count)
xdp_ring->next_to_use = 0;
}
if (tx_desc) {
ixgbe_xdp_ring_update_tail(xdp_ring);
xsk_umem_consume_tx_done(xdp_ring->xsk_umem);
}
return !!budget && work_done;
}
void dna_ixgbe_alloc_tx_buffers(struct ixgbe_ring *tx_ring, struct pfring_hooks *hook) {
union ixgbe_adv_tx_desc *tx_desc, *shadow_tx_desc;
struct ixgbe_tx_buffer *bi;
u16 i;
int num_slots_per_page = tx_ring->dna.tot_packet_memory / tx_ring->dna.packet_slot_len;
// struct ixgbe_adapter *adapter = netdev_priv(tx_ring->netdev);
/* Check if the memory has been already allocated */
if(tx_ring->dna.memory_allocated) return;
/* nothing to do or no valid netdev defined */
if (!netdev_ring(tx_ring))
return;
/* We suppose that RX and TX are in sync */
if(unlikely(enable_debug))
printk("%s(): tx_ring->dna.rx_tx.tx.tot_packet_memory=%d dna.num_memory_pages=%d\n",
__FUNCTION__, tx_ring->dna.tot_packet_memory, tx_ring->dna.num_memory_pages);
for(i=0; i<tx_ring->dna.num_memory_pages; i++) {
tx_ring->dna.rx_tx.tx.packet_memory[i] =
alloc_contiguous_memory(&tx_ring->dna.tot_packet_memory,
&tx_ring->dna.mem_order,
tx_ring->q_vector->numa_node);
if (tx_ring->dna.rx_tx.tx.packet_memory[i] == 0) {
printk("\n\n%s() ERROR: not enough memory for TX DMA ring!!\n\n\n",
__FUNCTION__);
return;
}
if(unlikely(enable_debug))
printk("[DNA] %s(): Successfully allocated TX %u@%u bytes at "
"0x%08lx [slot_len=%d]\n",__FUNCTION__,
tx_ring->dna.tot_packet_memory, i,
tx_ring->dna.rx_tx.tx.packet_memory[i],
tx_ring->dna.packet_slot_len);
}
if(unlikely(enable_debug))
printk("[DNA] %s(): %s@%d ptr=%p memory allocated on node %d\n", __FUNCTION__,
tx_ring->netdev->name, tx_ring->queue_index, tx_ring, tx_ring->q_vector->numa_node);
for(i=0; i < tx_ring->count; i++) {
u_int offset, page_index;
char *pkt;
page_index = i / num_slots_per_page;
offset = (i % num_slots_per_page) * tx_ring->dna.packet_slot_len;
pkt = (char *)(tx_ring->dna.rx_tx.tx.packet_memory[page_index] + offset);
bi = &tx_ring->tx_buffer_info[i];
bi->skb = NULL;
tx_desc = IXGBE_TX_DESC(tx_ring, i);
if(unlikely(enable_debug))
printk("%s(): Mapping TX slot %d of %d [pktaddr=%p][tx_desc=%p][offset=%u]\n",
__FUNCTION__, i, tx_ring->dna.packet_num_slots,
pkt, tx_desc, offset);
bi->dma = pci_map_single(to_pci_dev(tx_ring->dev), pkt,
tx_ring->dna.packet_slot_len,
PCI_DMA_BIDIRECTIONAL /* PCI_DMA_TODEVICE */ );
tx_desc->read.buffer_addr = cpu_to_le64(bi->dma);
shadow_tx_desc = IXGBE_TX_DESC(tx_ring, i + tx_ring->count);
memcpy(shadow_tx_desc, tx_desc, sizeof(union ixgbe_adv_tx_desc));
} /* for */
tx_ring->dna.memory_allocated = 1;
}
bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector,
struct ixgbe_ring *tx_ring, int napi_budget)
{
unsigned int total_packets = 0, total_bytes = 0;
u32 i = tx_ring->next_to_clean, xsk_frames = 0;
unsigned int budget = q_vector->tx.work_limit;
struct xdp_umem *umem = tx_ring->xsk_umem;
union ixgbe_adv_tx_desc *tx_desc;
struct ixgbe_tx_buffer *tx_bi;
bool xmit_done;
tx_bi = &tx_ring->tx_buffer_info[i];
tx_desc = IXGBE_TX_DESC(tx_ring, i);
i -= tx_ring->count;
do {
if (!(tx_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
break;
total_bytes += tx_bi->bytecount;
total_packets += tx_bi->gso_segs;
if (tx_bi->xdpf)
ixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi);
else
xsk_frames++;
tx_bi->xdpf = NULL;
total_bytes += tx_bi->bytecount;
tx_bi++;
tx_desc++;
i++;
if (unlikely(!i)) {
i -= tx_ring->count;
tx_bi = tx_ring->tx_buffer_info;
tx_desc = IXGBE_TX_DESC(tx_ring, 0);
}
/* issue prefetch for next Tx descriptor */
prefetch(tx_desc);
/* update budget accounting */
budget--;
} while (likely(budget));
i += tx_ring->count;
tx_ring->next_to_clean = i;
u64_stats_update_begin(&tx_ring->syncp);
tx_ring->stats.bytes += total_bytes;
tx_ring->stats.packets += total_packets;
u64_stats_update_end(&tx_ring->syncp);
q_vector->tx.total_bytes += total_bytes;
q_vector->tx.total_packets += total_packets;
if (xsk_frames)
xsk_umem_complete_tx(umem, xsk_frames);
xmit_done = ixgbe_xmit_zc(tx_ring, q_vector->tx.work_limit);
return budget > 0 && xmit_done;
}
