/* The higher levels take care of making this non-reentrant (it's
* called with bh's disabled).
*/
static netdev_tx_t loopback_xmit(struct sk_buff *skb,
struct net_device *dev)
{
struct pcpu_lstats *lb_stats;
int len;
skb_tx_timestamp(skb);
skb_orphan(skb);
/* Before queueing this packet to netif_rx(),
* make sure dst is refcounted.
*/
skb_dst_force(skb);
skb->protocol = eth_type_trans(skb, dev);
/* it's OK to use per_cpu_ptr() because BHs are off */
lb_stats = this_cpu_ptr(dev->lstats);
len = skb->len;
if (likely(netif_rx(skb) == NET_RX_SUCCESS)) {
u64_stats_update_begin(&lb_stats->syncp);
lb_stats->bytes += len;
lb_stats->packets++;
u64_stats_update_end(&lb_stats->syncp);
}
return NETDEV_TX_OK;
}
static int hip04_mac_start_xmit(struct sk_buff *skb, struct net_device *ndev)
{
struct hip04_priv *priv = netdev_priv(ndev);
struct net_device_stats *stats = &ndev->stats;
unsigned int tx_head = priv->tx_head, count;
struct tx_desc *desc = &priv->tx_desc[tx_head];
dma_addr_t phys;
smp_rmb();
count = tx_count(tx_head, ACCESS_ONCE(priv->tx_tail));
if (count == (TX_DESC_NUM - 1)) {
netif_stop_queue(ndev);
return NETDEV_TX_BUSY;
}
phys = dma_map_single(&ndev->dev, skb->data, skb->len, DMA_TO_DEVICE);
if (dma_mapping_error(&ndev->dev, phys)) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
priv->tx_skb[tx_head] = skb;
priv->tx_phys[tx_head] = phys;
desc->send_addr = cpu_to_be32(phys);
desc->send_size = cpu_to_be32(skb->len);
desc->cfg = cpu_to_be32(TX_CLEAR_WB | TX_FINISH_CACHE_INV);
phys = priv->tx_desc_dma + tx_head * sizeof(struct tx_desc);
desc->wb_addr = cpu_to_be32(phys);
skb_tx_timestamp(skb);
hip04_set_xmit_desc(priv, phys);
priv->tx_head = TX_NEXT(tx_head);
count++;
netdev_sent_queue(ndev, skb->len);
stats->tx_bytes += skb->len;
stats->tx_packets++;
/* Ensure tx_head update visible to tx reclaim */
smp_wmb();
/* queue is getting full, better start cleaning up now */
if (count >= priv->tx_coalesce_frames) {
if (napi_schedule_prep(&priv->napi)) {
/* disable rx interrupt and timer */
priv->reg_inten &= ~(RCV_INT);
writel_relaxed(DEF_INT_MASK & ~RCV_INT,
priv->base + PPE_INTEN);
hrtimer_cancel(&priv->tx_coalesce_timer);
__napi_schedule(&priv->napi);
}
} else if (!hrtimer_is_queued(&priv->tx_coalesce_timer)) {
/* cleanup not pending yet, start a new timer */
hip04_start_tx_timer(priv);
}
return NETDEV_TX_OK;
}
/**
* arc_emac_tx - Starts the data transmission.
* @skb: sk_buff pointer that contains data to be Transmitted.
* @ndev: Pointer to net_device structure.
*
* returns: NETDEV_TX_OK, on success
* NETDEV_TX_BUSY, if any of the descriptors are not free.
*
* This function is invoked from upper layers to initiate transmission.
*/
static int arc_emac_tx(struct sk_buff *skb, struct net_device *ndev)
{
struct arc_emac_priv *priv = netdev_priv(ndev);
unsigned int len, *txbd_curr = &priv->txbd_curr;
struct net_device_stats *stats = &priv->stats;
__le32 *info = &priv->txbd[*txbd_curr].info;
dma_addr_t addr;
if (skb_padto(skb, ETH_ZLEN))
return NETDEV_TX_OK;
len = max_t(unsigned int, ETH_ZLEN, skb->len);
/* EMAC still holds this buffer in its possession.
* CPU must not modify this buffer descriptor
*/
if (unlikely((le32_to_cpu(*info) & OWN_MASK) == FOR_EMAC)) {
netif_stop_queue(ndev);
return NETDEV_TX_BUSY;
}
addr = dma_map_single(&ndev->dev, (void *)skb->data, len,
DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(&ndev->dev, addr))) {
stats->tx_dropped++;
stats->tx_errors++;
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
dma_unmap_addr_set(&priv->tx_buff[*txbd_curr], addr, addr);
dma_unmap_len_set(&priv->tx_buff[*txbd_curr], len, len);
priv->tx_buff[*txbd_curr].skb = skb;
priv->txbd[*txbd_curr].data = cpu_to_le32(addr);
/* Make sure pointer to data buffer is set */
wmb();
skb_tx_timestamp(skb);
*info = cpu_to_le32(FOR_EMAC | FIRST_OR_LAST_MASK | len);
/* Increment index to point to the next BD */
*txbd_curr = (*txbd_curr + 1) % TX_BD_NUM;
/* Get "info" of the next BD */
info = &priv->txbd[*txbd_curr].info;
/* Check if if Tx BD ring is full - next BD is still owned by EMAC */
if (unlikely((le32_to_cpu(*info) & OWN_MASK) == FOR_EMAC))
netif_stop_queue(ndev);
arc_reg_set(priv, R_STATUS, TXPL_MASK);
return NETDEV_TX_OK;
}
static netdev_tx_t xge_start_xmit(struct sk_buff *skb, struct net_device *ndev)
{
struct xge_pdata *pdata = netdev_priv(ndev);
struct device *dev = &pdata->pdev->dev;
struct xge_desc_ring *tx_ring;
struct xge_raw_desc *raw_desc;
static dma_addr_t dma_addr;
u64 addr_lo, addr_hi;
void *pkt_buf;
u8 tail;
u16 len;
tx_ring = pdata->tx_ring;
tail = tx_ring->tail;
len = skb_headlen(skb);
raw_desc = &tx_ring->raw_desc[tail];
if (!is_tx_slot_available(raw_desc)) {
netif_stop_queue(ndev);
return NETDEV_TX_BUSY;
}
/* Packet buffers should be 64B aligned */
pkt_buf = dma_zalloc_coherent(dev, XGENE_ENET_STD_MTU, &dma_addr,
GFP_ATOMIC);
if (unlikely(!pkt_buf)) {
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
memcpy(pkt_buf, skb->data, len);
addr_hi = GET_BITS(NEXT_DESC_ADDRH, le64_to_cpu(raw_desc->m1));
addr_lo = GET_BITS(NEXT_DESC_ADDRL, le64_to_cpu(raw_desc->m1));
raw_desc->m1 = cpu_to_le64(SET_BITS(NEXT_DESC_ADDRL, addr_lo) |
SET_BITS(NEXT_DESC_ADDRH, addr_hi) |
SET_BITS(PKT_ADDRH,
upper_32_bits(dma_addr)));
tx_ring->pkt_info[tail].skb = skb;
tx_ring->pkt_info[tail].dma_addr = dma_addr;
tx_ring->pkt_info[tail].pkt_buf = pkt_buf;
dma_wmb();
raw_desc->m0 = cpu_to_le64(SET_BITS(PKT_ADDRL, dma_addr) |
SET_BITS(PKT_SIZE, len) |
SET_BITS(E, 0));
skb_tx_timestamp(skb);
xge_wr_csr(pdata, DMATXCTRL, 1);
tx_ring->tail = (tail + 1) & (XGENE_ENET_NUM_DESC - 1);
return NETDEV_TX_OK;
}
static netdev_tx_t pci_eth_start_xmit(struct sk_buff *skb,
struct net_device *dev)
{
struct pci_eth_private *priv = netdev_priv(dev);
struct pci_eth_descriptor *descptr;
void __iomem *ioaddr = dev->base_addr;
unsigned long flags;
/* Critical Section */
spin_lock_irqsave(&priv->lock, flags);
/* TX resource check */
if (!priv->tx_free_desc) {
spin_unlock_irqrestore(&priv->lock, flags);
netif_stop_queue(dev);
netdev_err(dev, ": no tx descriptor\n");
return NETDEV_TX_BUSY;
}
/* Statistic Counter */
dev->stats.tx_packets++;
dev->stats.tx_bytes += skb->len;
/* Decrement free descriptors counter */
priv->tx_free_desc--;
/* Set TX descriptor & Transmit it */
descptr = priv->tx_insert_ptr;
if (skb->len < ETH_ZLEN)
descptr->len = ETH_ZLEN;
else
descptr->len = skb->len;
descptr->skb_ptr = skb;
descptr->buf = cpu_to_le32(pci_map_single(priv->pdev,
skb->data, skb->len, PCI_DMA_TODEVICE));
descptr->status = DSC_OWNER_MAC;
skb_tx_timestamp(skb);
/* TODO: Trigger the MAC to check the TX descriptor - start DMA
* transaction.
*/
/* After DMA transaction perform the following check */
/* If no tx resource, stop */
if (!priv->tx_free_desc)
netif_stop_queue(dev);
spin_unlock_irqrestore(&priv->lock, flags);
return NETDEV_TX_OK;
}
static netdev_tx_t r6040_start_xmit(struct sk_buff *skb,
struct net_device *dev)
{
struct r6040_private *lp = netdev_priv(dev);
struct r6040_descriptor *descptr;
void __iomem *ioaddr = lp->base;
unsigned long flags;
/* Critical Section */
spin_lock_irqsave(&lp->lock, flags);
/* TX resource check */
if (!lp->tx_free_desc) {
spin_unlock_irqrestore(&lp->lock, flags);
netif_stop_queue(dev);
netdev_err(dev, ": no tx descriptor\n");
return NETDEV_TX_BUSY;
}
/* Statistic Counter */
dev->stats.tx_packets++;
dev->stats.tx_bytes += skb->len;
/* Set TX descriptor & Transmit it */
lp->tx_free_desc--;
descptr = lp->tx_insert_ptr;
if (skb->len < MISR)
descptr->len = MISR;
else
descptr->len = skb->len;
descptr->skb_ptr = skb;
descptr->buf = cpu_to_le32(pci_map_single(lp->pdev,
skb->data, skb->len, PCI_DMA_TODEVICE));
descptr->status = DSC_OWNER_MAC;
skb_tx_timestamp(skb);
/* Trigger the MAC to check the TX descriptor */
iowrite16(TM2TX, ioaddr + MTPR);
lp->tx_insert_ptr = descptr->vndescp;
/* If no tx resource, stop */
if (!lp->tx_free_desc)
netif_stop_queue(dev);
spin_unlock_irqrestore(&lp->lock, flags);
return NETDEV_TX_OK;
}
static int fs_enet_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct fs_enet_private *fep = netdev_priv(dev);
cbd_t __iomem *bdp;
int curidx;
u16 sc;
unsigned long flags;
#ifdef CONFIG_FS_ENET_MPC5121_FEC
if (((unsigned long)skb->data) & 0x3) {
skb = tx_skb_align_workaround(dev, skb);
if (!skb) {
/*
* We have lost packet due to memory allocation error
* in tx_skb_align_workaround(). Hopefully original
* skb is still valid, so try transmit it later.
*/
return NETDEV_TX_BUSY;
}
}
#endif
spin_lock_irqsave(&fep->tx_lock, flags);
/*
* Fill in a Tx ring entry
*/
bdp = fep->cur_tx;
if (!fep->tx_free || (CBDR_SC(bdp) & BD_ENET_TX_READY)) {
netif_stop_queue(dev);
spin_unlock_irqrestore(&fep->tx_lock, flags);
/*
* Ooops. All transmit buffers are full. Bail out.
* This should not happen, since the tx queue should be stopped.
*/
dev_warn(fep->dev, "tx queue full!.\n");
return NETDEV_TX_BUSY;
}
curidx = bdp - fep->tx_bd_base;
/*
* Clear all of the status flags.
*/
CBDC_SC(bdp, BD_ENET_TX_STATS);
/*
* Save skb pointer.
*/
fep->tx_skbuff[curidx] = skb;
fep->stats.tx_bytes += skb->len;
/*
* Push the data cache so the CPM does not get stale memory data.
*/
CBDW_BUFADDR(bdp, dma_map_single(fep->dev,
skb->data, skb->len, DMA_TO_DEVICE));
CBDW_DATLEN(bdp, skb->len);
/*
* If this was the last BD in the ring, start at the beginning again.
*/
if ((CBDR_SC(bdp) & BD_ENET_TX_WRAP) == 0)
fep->cur_tx++;
else
fep->cur_tx = fep->tx_bd_base;
if (!--fep->tx_free)
netif_stop_queue(dev);
/* Trigger transmission start */
sc = BD_ENET_TX_READY | BD_ENET_TX_INTR |
BD_ENET_TX_LAST | BD_ENET_TX_TC;
/* note that while FEC does not have this bit
* it marks it as available for software use
* yay for hw reuse :) */
if (skb->len <= 60)
sc |= BD_ENET_TX_PAD;
CBDS_SC(bdp, sc);
skb_tx_timestamp(skb);
(*fep->ops->tx_kickstart)(dev);
spin_unlock_irqrestore(&fep->tx_lock, flags);
return NETDEV_TX_OK;
}
/**
* nfp_net_tx() - Main transmit entry point
* @skb: SKB to transmit
* @netdev: netdev structure
*
* Return: NETDEV_TX_OK on success.
*/
static int nfp_net_tx(struct sk_buff *skb, struct net_device *netdev)
{
struct nfp_net *nn = netdev_priv(netdev);
const struct skb_frag_struct *frag;
struct nfp_net_r_vector *r_vec;
struct nfp_net_tx_desc *txd, txdg;
struct nfp_net_tx_buf *txbuf;
struct nfp_net_tx_ring *tx_ring;
struct netdev_queue *nd_q;
dma_addr_t dma_addr;
unsigned int fsize;
int f, nr_frags;
int wr_idx;
u16 qidx;
qidx = skb_get_queue_mapping(skb);
tx_ring = &nn->tx_rings[qidx];
r_vec = tx_ring->r_vec;
nd_q = netdev_get_tx_queue(nn->netdev, qidx);
nr_frags = skb_shinfo(skb)->nr_frags;
if (unlikely(nfp_net_tx_full(tx_ring, nr_frags + 1))) {
nn_warn_ratelimit(nn, "TX ring %d busy. wrp=%u rdp=%u\n",
qidx, tx_ring->wr_p, tx_ring->rd_p);
netif_tx_stop_queue(nd_q);
u64_stats_update_begin(&r_vec->tx_sync);
r_vec->tx_busy++;
u64_stats_update_end(&r_vec->tx_sync);
return NETDEV_TX_BUSY;
}
/* Start with the head skbuf */
dma_addr = dma_map_single(&nn->pdev->dev, skb->data, skb_headlen(skb),
DMA_TO_DEVICE);
if (dma_mapping_error(&nn->pdev->dev, dma_addr))
goto err_free;
wr_idx = tx_ring->wr_p % tx_ring->cnt;
/* Stash the soft descriptor of the head then initialize it */
txbuf = &tx_ring->txbufs[wr_idx];
txbuf->skb = skb;
txbuf->dma_addr = dma_addr;
txbuf->fidx = -1;
txbuf->pkt_cnt = 1;
txbuf->real_len = skb->len;
/* Build TX descriptor */
txd = &tx_ring->txds[wr_idx];
txd->offset_eop = (nr_frags == 0) ? PCIE_DESC_TX_EOP : 0;
txd->dma_len = cpu_to_le16(skb_headlen(skb));
nfp_desc_set_dma_addr(txd, dma_addr);
txd->data_len = cpu_to_le16(skb->len);
txd->flags = 0;
txd->mss = 0;
txd->l4_offset = 0;
nfp_net_tx_tso(nn, r_vec, txbuf, txd, skb);
nfp_net_tx_csum(nn, r_vec, txbuf, txd, skb);
if (skb_vlan_tag_present(skb) && nn->ctrl & NFP_NET_CFG_CTRL_TXVLAN) {
txd->flags |= PCIE_DESC_TX_VLAN;
txd->vlan = cpu_to_le16(skb_vlan_tag_get(skb));
}
/* Gather DMA */
if (nr_frags > 0) {
/* all descs must match except for in addr, length and eop */
txdg = *txd;
for (f = 0; f < nr_frags; f++) {
frag = &skb_shinfo(skb)->frags[f];
fsize = skb_frag_size(frag);
dma_addr = skb_frag_dma_map(&nn->pdev->dev, frag, 0,
fsize, DMA_TO_DEVICE);
if (dma_mapping_error(&nn->pdev->dev, dma_addr))
goto err_unmap;
wr_idx = (wr_idx + 1) % tx_ring->cnt;
tx_ring->txbufs[wr_idx].skb = skb;
tx_ring->txbufs[wr_idx].dma_addr = dma_addr;
tx_ring->txbufs[wr_idx].fidx = f;
txd = &tx_ring->txds[wr_idx];
*txd = txdg;
txd->dma_len = cpu_to_le16(fsize);
nfp_desc_set_dma_addr(txd, dma_addr);
txd->offset_eop =
(f == nr_frags - 1) ? PCIE_DESC_TX_EOP : 0;
}
u64_stats_update_begin(&r_vec->tx_sync);
r_vec->tx_gather++;
u64_stats_update_end(&r_vec->tx_sync);
}
netdev_tx_sent_queue(nd_q, txbuf->real_len);
tx_ring->wr_p += nr_frags + 1;
if (nfp_net_tx_ring_should_stop(tx_ring))
nfp_net_tx_ring_stop(nd_q, tx_ring);
tx_ring->wr_ptr_add += nr_frags + 1;
if (!skb->xmit_more || netif_xmit_stopped(nd_q)) {
/* force memory write before we let HW know */
wmb();
nfp_qcp_wr_ptr_add(tx_ring->qcp_q, tx_ring->wr_ptr_add);
tx_ring->wr_ptr_add = 0;
}
skb_tx_timestamp(skb);
return NETDEV_TX_OK;
err_unmap:
--f;
while (f >= 0) {
frag = &skb_shinfo(skb)->frags[f];
dma_unmap_page(&nn->pdev->dev,
tx_ring->txbufs[wr_idx].dma_addr,
skb_frag_size(frag), DMA_TO_DEVICE);
tx_ring->txbufs[wr_idx].skb = NULL;
tx_ring->txbufs[wr_idx].dma_addr = 0;
tx_ring->txbufs[wr_idx].fidx = -2;
wr_idx = wr_idx - 1;
if (wr_idx < 0)
wr_idx += tx_ring->cnt;
}
dma_unmap_single(&nn->pdev->dev, tx_ring->txbufs[wr_idx].dma_addr,
skb_headlen(skb), DMA_TO_DEVICE);
tx_ring->txbufs[wr_idx].skb = NULL;
tx_ring->txbufs[wr_idx].dma_addr = 0;
tx_ring->txbufs[wr_idx].fidx = -2;
err_free:
nn_warn_ratelimit(nn, "Failed to map DMA TX buffer\n");
u64_stats_update_begin(&r_vec->tx_sync);
r_vec->tx_errors++;
u64_stats_update_end(&r_vec->tx_sync);
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
static netdev_tx_t
fec_enet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
{
struct fec_enet_private *fep = netdev_priv(ndev);
const struct platform_device_id *id_entry =
platform_get_device_id(fep->pdev);
struct bufdesc *bdp;
void *bufaddr;
unsigned short status;
unsigned long flags;
if (!fep->link) {
/* Link is down or autonegotiation is in progress. */
return NETDEV_TX_BUSY;
}
spin_lock_irqsave(&fep->hw_lock, flags);
/* Fill in a Tx ring entry */
bdp = fep->cur_tx;
status = bdp->cbd_sc;
if (status & BD_ENET_TX_READY) {
/* Ooops. All transmit buffers are full. Bail out.
* This should not happen, since ndev->tbusy should be set.
*/
printk("%s: tx queue full!.\n", ndev->name);
spin_unlock_irqrestore(&fep->hw_lock, flags);
return NETDEV_TX_BUSY;
}
/* Clear all of the status flags */
status &= ~BD_ENET_TX_STATS;
/* Set buffer length and buffer pointer */
bufaddr = skb->data;
bdp->cbd_datlen = skb->len;
/*
* On some FEC implementations data must be aligned on
* 4-byte boundaries. Use bounce buffers to copy data
* and get it aligned. Ugh.
*/
if (((unsigned long) bufaddr) & FEC_ALIGNMENT) {
unsigned int index;
if (fep->bufdesc_ex)
index = (struct bufdesc_ex *)bdp -
(struct bufdesc_ex *)fep->tx_bd_base;
else
index = bdp - fep->tx_bd_base;
memcpy(fep->tx_bounce[index], skb->data, skb->len);
bufaddr = fep->tx_bounce[index];
}
/*
* Some design made an incorrect assumption on endian mode of
* the system that it's running on. As the result, driver has to
* swap every frame going to and coming from the controller.
*/
if (id_entry->driver_data & FEC_QUIRK_SWAP_FRAME)
swap_buffer(bufaddr, skb->len);
/* Save skb pointer */
fep->tx_skbuff[fep->skb_cur] = skb;
ndev->stats.tx_bytes += skb->len;
fep->skb_cur = (fep->skb_cur+1) & TX_RING_MOD_MASK;
/* Push the data cache so the CPM does not get stale memory
* data.
*/
bdp->cbd_bufaddr = dma_map_single(&fep->pdev->dev, bufaddr,
FEC_ENET_TX_FRSIZE, DMA_TO_DEVICE);
/* Send it on its way. Tell FEC it's ready, interrupt when done,
* it's the last BD of the frame, and to put the CRC on the end.
*/
status |= (BD_ENET_TX_READY | BD_ENET_TX_INTR
| BD_ENET_TX_LAST | BD_ENET_TX_TC);
bdp->cbd_sc = status;
if (fep->bufdesc_ex) {
struct bufdesc_ex *ebdp = (struct bufdesc_ex *)bdp;
ebdp->cbd_bdu = 0;
if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
fep->hwts_tx_en)) {
ebdp->cbd_esc = (BD_ENET_TX_TS | BD_ENET_TX_INT);
skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
} else {
ebdp->cbd_esc = BD_ENET_TX_INT;
}
}
/* Trigger transmission start */
writel(0, fep->hwp + FEC_X_DES_ACTIVE);
/* If this was the last BD in the ring, start at the beginning again. */
if (status & BD_ENET_TX_WRAP)
bdp = fep->tx_bd_base;
else
bdp = fec_enet_get_nextdesc(bdp, fep->bufdesc_ex);
if (bdp == fep->dirty_tx) {
fep->tx_full = 1;
netif_stop_queue(ndev);
}
fep->cur_tx = bdp;
skb_tx_timestamp(skb);
spin_unlock_irqrestore(&fep->hw_lock, flags);
return NETDEV_TX_OK;
}
static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
{
struct net_device_context *net_device_ctx = netdev_priv(net);
struct hv_netvsc_packet *packet = NULL;
int ret;
unsigned int num_data_pgs;
struct rndis_message *rndis_msg;
struct rndis_packet *rndis_pkt;
u32 rndis_msg_size;
struct rndis_per_packet_info *ppi;
struct ndis_tcp_ip_checksum_info *csum_info;
int hdr_offset;
u32 net_trans_info;
u32 hash;
u32 skb_length;
struct hv_page_buffer page_buf[MAX_PAGE_BUFFER_COUNT];
struct hv_page_buffer *pb = page_buf;
/* We will atmost need two pages to describe the rndis
* header. We can only transmit MAX_PAGE_BUFFER_COUNT number
* of pages in a single packet. If skb is scattered around
* more pages we try linearizing it.
*/
skb_length = skb->len;
num_data_pgs = netvsc_get_slots(skb) + 2;
if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
++net_device_ctx->eth_stats.tx_scattered;
if (skb_linearize(skb))
goto no_memory;
num_data_pgs = netvsc_get_slots(skb) + 2;
if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
++net_device_ctx->eth_stats.tx_too_big;
goto drop;
}
}
/*
* Place the rndis header in the skb head room and
* the skb->cb will be used for hv_netvsc_packet
* structure.
*/
ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
if (ret)
goto no_memory;
/* Use the skb control buffer for building up the packet */
BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
FIELD_SIZEOF(struct sk_buff, cb));
packet = (struct hv_netvsc_packet *)skb->cb;
packet->q_idx = skb_get_queue_mapping(skb);
packet->total_data_buflen = skb->len;
rndis_msg = (struct rndis_message *)skb->head;
memset(rndis_msg, 0, RNDIS_AND_PPI_SIZE);
/* Add the rndis header */
rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
rndis_msg->msg_len = packet->total_data_buflen;
rndis_pkt = &rndis_msg->msg.pkt;
rndis_pkt->data_offset = sizeof(struct rndis_packet);
rndis_pkt->data_len = packet->total_data_buflen;
rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet);
rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
hash = skb_get_hash_raw(skb);
if (hash != 0 && net->real_num_tx_queues > 1) {
rndis_msg_size += NDIS_HASH_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
NBL_HASH_VALUE);
*(u32 *)((void *)ppi + ppi->ppi_offset) = hash;
}
if (skb_vlan_tag_present(skb)) {
struct ndis_pkt_8021q_info *vlan;
rndis_msg_size += NDIS_VLAN_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
IEEE_8021Q_INFO);
vlan = (struct ndis_pkt_8021q_info *)((void *)ppi +
ppi->ppi_offset);
vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
VLAN_PRIO_SHIFT;
}
net_trans_info = get_net_transport_info(skb, &hdr_offset);
/*
* Setup the sendside checksum offload only if this is not a
* GSO packet.
*/
if ((net_trans_info & (INFO_TCP | INFO_UDP)) && skb_is_gso(skb)) {
struct ndis_tcp_lso_info *lso_info;
rndis_msg_size += NDIS_LSO_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
TCP_LARGESEND_PKTINFO);
lso_info = (struct ndis_tcp_lso_info *)((void *)ppi +
ppi->ppi_offset);
lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
if (net_trans_info & (INFO_IPV4 << 16)) {
lso_info->lso_v2_transmit.ip_version =
NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
ip_hdr(skb)->tot_len = 0;
ip_hdr(skb)->check = 0;
tcp_hdr(skb)->check =
~csum_tcpudp_magic(ip_hdr(skb)->saddr,
ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
} else {
lso_info->lso_v2_transmit.ip_version =
NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
ipv6_hdr(skb)->payload_len = 0;
tcp_hdr(skb)->check =
~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
&ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
}
lso_info->lso_v2_transmit.tcp_header_offset = hdr_offset;
lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
if (net_trans_info & INFO_TCP) {
rndis_msg_size += NDIS_CSUM_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
TCPIP_CHKSUM_PKTINFO);
csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi +
ppi->ppi_offset);
if (net_trans_info & (INFO_IPV4 << 16))
csum_info->transmit.is_ipv4 = 1;
else
csum_info->transmit.is_ipv6 = 1;
csum_info->transmit.tcp_checksum = 1;
csum_info->transmit.tcp_header_offset = hdr_offset;
} else {
/* UDP checksum (and other) offload is not supported. */
if (skb_checksum_help(skb))
goto drop;
}
}
/* Start filling in the page buffers with the rndis hdr */
rndis_msg->msg_len += rndis_msg_size;
packet->total_data_buflen = rndis_msg->msg_len;
packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
skb, packet, &pb);
/* timestamp packet in software */
skb_tx_timestamp(skb);
ret = netvsc_send(net_device_ctx->device_ctx, packet,
rndis_msg, &pb, skb);
if (likely(ret == 0)) {
struct netvsc_stats *tx_stats = this_cpu_ptr(net_device_ctx->tx_stats);
u64_stats_update_begin(&tx_stats->syncp);
tx_stats->packets++;
tx_stats->bytes += skb_length;
u64_stats_update_end(&tx_stats->syncp);
return NETDEV_TX_OK;
}
if (ret == -EAGAIN) {
++net_device_ctx->eth_stats.tx_busy;
return NETDEV_TX_BUSY;
}
if (ret == -ENOSPC)
++net_device_ctx->eth_stats.tx_no_space;
drop:
dev_kfree_skb_any(skb);
net->stats.tx_dropped++;
return NETDEV_TX_OK;
no_memory:
++net_device_ctx->eth_stats.tx_no_memory;
goto drop;
}
static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
{
struct net_device_context *net_device_ctx = netdev_priv(net);
struct hv_netvsc_packet *packet = NULL;
int ret;
unsigned int num_data_pgs;
struct rndis_message *rndis_msg;
struct rndis_packet *rndis_pkt;
u32 rndis_msg_size;
bool isvlan;
bool linear = false;
struct rndis_per_packet_info *ppi;
struct ndis_tcp_ip_checksum_info *csum_info;
struct ndis_tcp_lso_info *lso_info;
int hdr_offset;
u32 net_trans_info;
u32 hash;
u32 skb_length;
struct hv_page_buffer page_buf[MAX_PAGE_BUFFER_COUNT];
struct hv_page_buffer *pb = page_buf;
struct netvsc_stats *tx_stats = this_cpu_ptr(net_device_ctx->tx_stats);
/* We will atmost need two pages to describe the rndis
* header. We can only transmit MAX_PAGE_BUFFER_COUNT number
* of pages in a single packet. If skb is scattered around
* more pages we try linearizing it.
*/
check_size:
skb_length = skb->len;
num_data_pgs = netvsc_get_slots(skb) + 2;
if (num_data_pgs > MAX_PAGE_BUFFER_COUNT && linear) {
net_alert_ratelimited("packet too big: %u pages (%u bytes)\n",
num_data_pgs, skb->len);
ret = -EFAULT;
goto drop;
} else if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
if (skb_linearize(skb)) {
net_alert_ratelimited("failed to linearize skb\n");
ret = -ENOMEM;
goto drop;
}
linear = true;
goto check_size;
}
/*
* Place the rndis header in the skb head room and
* the skb->cb will be used for hv_netvsc_packet
* structure.
*/
ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
if (ret) {
netdev_err(net, "unable to alloc hv_netvsc_packet\n");
ret = -ENOMEM;
goto drop;
}
/* Use the skb control buffer for building up the packet */
BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
FIELD_SIZEOF(struct sk_buff, cb));
packet = (struct hv_netvsc_packet *)skb->cb;
packet->q_idx = skb_get_queue_mapping(skb);
packet->total_data_buflen = skb->len;
rndis_msg = (struct rndis_message *)skb->head;
memset(rndis_msg, 0, RNDIS_AND_PPI_SIZE);
isvlan = skb->vlan_tci & VLAN_TAG_PRESENT;
/* Add the rndis header */
rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
rndis_msg->msg_len = packet->total_data_buflen;
rndis_pkt = &rndis_msg->msg.pkt;
rndis_pkt->data_offset = sizeof(struct rndis_packet);
rndis_pkt->data_len = packet->total_data_buflen;
rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet);
rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
hash = skb_get_hash_raw(skb);
if (hash != 0 && net->real_num_tx_queues > 1) {
rndis_msg_size += NDIS_HASH_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
NBL_HASH_VALUE);
*(u32 *)((void *)ppi + ppi->ppi_offset) = hash;
}
if (isvlan) {
struct ndis_pkt_8021q_info *vlan;
rndis_msg_size += NDIS_VLAN_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
IEEE_8021Q_INFO);
vlan = (struct ndis_pkt_8021q_info *)((void *)ppi +
ppi->ppi_offset);
vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
VLAN_PRIO_SHIFT;
}
net_trans_info = get_net_transport_info(skb, &hdr_offset);
if (net_trans_info == TRANSPORT_INFO_NOT_IP)
goto do_send;
/*
* Setup the sendside checksum offload only if this is not a
* GSO packet.
*/
if (skb_is_gso(skb))
goto do_lso;
if ((skb->ip_summed == CHECKSUM_NONE) ||
(skb->ip_summed == CHECKSUM_UNNECESSARY))
goto do_send;
rndis_msg_size += NDIS_CSUM_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
TCPIP_CHKSUM_PKTINFO);
csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi +
ppi->ppi_offset);
if (net_trans_info & (INFO_IPV4 << 16))
csum_info->transmit.is_ipv4 = 1;
else
csum_info->transmit.is_ipv6 = 1;
if (net_trans_info & INFO_TCP) {
csum_info->transmit.tcp_checksum = 1;
csum_info->transmit.tcp_header_offset = hdr_offset;
} else if (net_trans_info & INFO_UDP) {
/* UDP checksum offload is not supported on ws2008r2.
* Furthermore, on ws2012 and ws2012r2, there are some
* issues with udp checksum offload from Linux guests.
* (these are host issues).
* For now compute the checksum here.
*/
struct udphdr *uh;
u16 udp_len;
ret = skb_cow_head(skb, 0);
if (ret)
goto drop;
uh = udp_hdr(skb);
udp_len = ntohs(uh->len);
uh->check = 0;
uh->check = csum_tcpudp_magic(ip_hdr(skb)->saddr,
ip_hdr(skb)->daddr,
udp_len, IPPROTO_UDP,
csum_partial(uh, udp_len, 0));
if (uh->check == 0)
uh->check = CSUM_MANGLED_0;
csum_info->transmit.udp_checksum = 0;
}
goto do_send;
do_lso:
rndis_msg_size += NDIS_LSO_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
TCP_LARGESEND_PKTINFO);
lso_info = (struct ndis_tcp_lso_info *)((void *)ppi +
ppi->ppi_offset);
lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
if (net_trans_info & (INFO_IPV4 << 16)) {
lso_info->lso_v2_transmit.ip_version =
NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
ip_hdr(skb)->tot_len = 0;
ip_hdr(skb)->check = 0;
tcp_hdr(skb)->check =
~csum_tcpudp_magic(ip_hdr(skb)->saddr,
ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
} else {
lso_info->lso_v2_transmit.ip_version =
NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
ipv6_hdr(skb)->payload_len = 0;
tcp_hdr(skb)->check =
~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
&ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
}
lso_info->lso_v2_transmit.tcp_header_offset = hdr_offset;
lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
do_send:
/* Start filling in the page buffers with the rndis hdr */
rndis_msg->msg_len += rndis_msg_size;
packet->total_data_buflen = rndis_msg->msg_len;
packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
skb, packet, &pb);
/* timestamp packet in software */
skb_tx_timestamp(skb);
ret = netvsc_send(net_device_ctx->device_ctx, packet,
rndis_msg, &pb, skb);
drop:
if (ret == 0) {
u64_stats_update_begin(&tx_stats->syncp);
tx_stats->packets++;
tx_stats->bytes += skb_length;
u64_stats_update_end(&tx_stats->syncp);
} else {
if (ret != -EAGAIN) {
dev_kfree_skb_any(skb);
net->stats.tx_dropped++;
}
}
return (ret == -EAGAIN) ? NETDEV_TX_BUSY : NETDEV_TX_OK;
}
netdev_tx_t usbnet_start_xmit (struct sk_buff *skb,
struct net_device *net)
{
struct usbnet *dev = netdev_priv(net);
int length;
struct urb *urb = NULL;
struct skb_data *entry;
struct driver_info *info = dev->driver_info;
unsigned long flags;
int retval;
if (skb)
skb_tx_timestamp(skb);
// some devices want funky USB-level framing, for
// win32 driver (usually) and/or hardware quirks
if (info->tx_fixup) {
skb = info->tx_fixup (dev, skb, GFP_ATOMIC);
if (!skb) {
if (netif_msg_tx_err(dev)) {
netif_dbg(dev, tx_err, dev->net, "can't tx_fixup skb\n");
goto drop;
} else {
/* cdc_ncm collected packet; waits for more */
goto not_drop;
}
}
}
length = skb->len;
if (!(urb = usb_alloc_urb (0, GFP_ATOMIC))) {
netif_dbg(dev, tx_err, dev->net, "no urb\n");
goto drop;
}
entry = (struct skb_data *) skb->cb;
entry->urb = urb;
entry->dev = dev;
entry->state = tx_start;
entry->length = length;
usb_fill_bulk_urb (urb, dev->udev, dev->out,
skb->data, skb->len, tx_complete, skb);
/* don't assume the hardware handles USB_ZERO_PACKET
* NOTE: strictly conforming cdc-ether devices should expect
* the ZLP here, but ignore the one-byte packet.
* NOTE2: CDC NCM specification is different from CDC ECM when
* handling ZLP/short packets, so cdc_ncm driver will make short
* packet itself if needed.
*/
if (length % dev->maxpacket == 0) {
if (!(info->flags & FLAG_SEND_ZLP)) {
if (!(info->flags & FLAG_MULTI_PACKET)) {
urb->transfer_buffer_length++;
if (skb_tailroom(skb)) {
skb->data[skb->len] = 0;
__skb_put(skb, 1);
}
}
} else
urb->transfer_flags |= URB_ZERO_PACKET;
}
spin_lock_irqsave(&dev->txq.lock, flags);
retval = usb_autopm_get_interface_async(dev->intf);
if (retval < 0) {
spin_unlock_irqrestore(&dev->txq.lock, flags);
goto drop;
}
#ifdef CONFIG_PM
/* if this triggers the device is still a sleep */
if (test_bit(EVENT_DEV_ASLEEP, &dev->flags)) {
/* transmission will be done in resume */
usb_anchor_urb(urb, &dev->deferred);
/* no use to process more packets */
netif_stop_queue(net);
spin_unlock_irqrestore(&dev->txq.lock, flags);
netdev_dbg(dev->net, "Delaying transmission for resumption\n");
goto deferred;
}
#endif
switch ((retval = usb_submit_urb (urb, GFP_ATOMIC))) {
case -EPIPE:
netif_stop_queue (net);
usbnet_defer_kevent (dev, EVENT_TX_HALT);
usb_autopm_put_interface_async(dev->intf);
break;
default:
usb_autopm_put_interface_async(dev->intf);
netif_dbg(dev, tx_err, dev->net,
"tx: submit urb err %d\n", retval);
break;
case 0:
net->trans_start = jiffies;
__skb_queue_tail (&dev->txq, skb);
if (dev->txq.qlen >= TX_QLEN (dev))
netif_stop_queue (net);
}
spin_unlock_irqrestore (&dev->txq.lock, flags);
if (retval) {
netif_dbg(dev, tx_err, dev->net, "drop, code %d\n", retval);
drop:
dev->net->stats.tx_dropped++;
not_drop:
if (skb)
dev_kfree_skb_any (skb);
usb_free_urb (urb);
} else
netif_dbg(dev, tx_queued, dev->net,
"> tx, len %d, type 0x%x\n", length, skb->protocol);
#ifdef CONFIG_PM
deferred:
#endif
return NETDEV_TX_OK;
}
netdev_tx_t mpodp_start_xmit(struct sk_buff *skb,
struct net_device *netdev)
{
struct mpodp_if_priv *priv = netdev_priv(netdev);
struct mpodp_tx *tx;
struct dma_async_tx_descriptor *dma_txd;
struct mpodp_cache_entry *entry;
int ret;
uint8_t fifo_mode;
int16_t requested_engine;
struct mpodp_pkt_hdr *hdr;
uint32_t tx_autoloop_next;
uint32_t tx_submitted, tx_next, tx_done;
uint32_t tx_mppa_idx;
int qidx;
unsigned long flags = 0;
struct mpodp_txq *txq;
/* Fetch HW queue selected by the kernel */
qidx = skb_get_queue_mapping(skb);
txq = &priv->txqs[qidx];
if (atomic_read(&priv->reset) == 1) {
mpodp_clean_tx_unlocked(priv, txq, -1);
goto addr_error;
}
tx_submitted = atomic_read(&txq->submitted);
/* Compute txd id */
tx_next = (tx_submitted + 1);
if (tx_next == txq->size)
tx_next = 0;
/* MPPA H2C Entry to use */
tx_mppa_idx = atomic_read(&txq->autoloop_cur);
tx_done = atomic_read(&txq->done);
if (tx_done != tx_submitted &&
((txq->ring[tx_done].jiffies + msecs_to_jiffies(5) >= jiffies) ||
(tx_submitted < tx_done && tx_submitted + txq->size - tx_done >= TX_POLL_THRESHOLD) ||
(tx_submitted >= tx_done && tx_submitted - tx_done >= TX_POLL_THRESHOLD))) {
mpodp_clean_tx_unlocked(priv, txq, -1);
}
/* Check if there are txd available */
if (tx_next == atomic_read(&txq->done)) {
/* Ring is full */
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d]: ring full \n", txq->id);
netif_tx_stop_queue(txq->txq);
return NETDEV_TX_BUSY;
}
tx = &(txq->ring[tx_submitted]);
entry = &(txq->cache[tx_mppa_idx]);
/* take the time */
mppa_pcie_time_get(priv->tx_time, &tx->time);
/* configure channel */
tx->dst_addr = entry->addr;
/* Check the provided address */
ret =
mppa_pcie_dma_check_addr(priv->pdata, tx->dst_addr, &fifo_mode,
&requested_engine);
if (ret) {
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: invalid send address %llx\n",
txq->id, tx_submitted, tx->dst_addr);
goto addr_error;
}
if (!fifo_mode) {
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: %llx is not a PCI2Noc addres\n",
txq->id, tx_submitted, tx->dst_addr);
goto addr_error;
}
if (requested_engine >= MPODP_NOC_CHAN_COUNT) {
if (netif_msg_tx_err(priv))
netdev_err(netdev,
"txq[%d] tx[%d]: address %llx using NoC engine out of range (%d >= %d)\n",
txq->id, tx_submitted, tx->dst_addr,
requested_engine, MPODP_NOC_CHAN_COUNT);
goto addr_error;
}
tx->chanidx = requested_engine;
/* The packet needs a header to determine size,timestamp, etc.
* Add it */
if (skb_headroom(skb) < sizeof(struct mpodp_pkt_hdr)) {
struct sk_buff *skb_new;
skb_new =
skb_realloc_headroom(skb, sizeof(struct mpodp_pkt_hdr));
if (!skb_new) {
netdev->stats.tx_errors++;
kfree_skb(skb);
return NETDEV_TX_OK;
}
kfree_skb(skb);
skb = skb_new;
}
hdr = (struct mpodp_pkt_hdr *)
skb_push(skb, sizeof(struct mpodp_pkt_hdr));
hdr->timestamp = priv->packet_id;
hdr->info._.pkt_id = priv->packet_id;
hdr->info.dword = 0ULL;
hdr->info._.pkt_size = skb->len; /* Also count the header size */
hdr->info._.pkt_id = priv->packet_id;
priv->packet_id++;
/* save skb to free it later */
tx->skb = skb;
tx->len = skb->len;
/* prepare sg */
if (map_skb(&priv->pdev->dev, skb, tx)){
if (netif_msg_tx_err(priv))
netdev_err(netdev, "tx %d: failed to map skb to dma\n",
tx_submitted);
goto busy;
}
if (priv->n_txqs > MPODP_NOC_CHAN_COUNT)
spin_lock_irqsave(&priv->tx_lock[requested_engine], flags);
/* Prepare slave args */
priv->tx_config[requested_engine].cfg.dst_addr = tx->dst_addr;
priv->tx_config[requested_engine].requested_engine = requested_engine;
/* FIFO mode, direction, latency were filled at setup */
if (dmaengine_slave_config(priv->tx_chan[requested_engine],
&priv->tx_config[requested_engine].cfg)) {
/* board has reset, wait for reset of netdev */
netif_tx_stop_queue(txq->txq);
netif_carrier_off(netdev);
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: cannot configure channel\n",
txq->id, tx_submitted);
goto busy;
}
/* get transfer descriptor */
dma_txd =
dmaengine_prep_slave_sg(priv->tx_chan[requested_engine], tx->sg,
tx->sg_len, DMA_MEM_TO_DEV, 0);
if (dma_txd == NULL) {
/* dmaengine_prep_slave_sg failed, retry */
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: cannot get dma descriptor\n",
txq->id, tx_submitted);
goto busy;
}
if (netif_msg_tx_queued(priv))
netdev_info(netdev,
"txq[%d] tx[%d]: transfer start (submitted: %d done: %d) len=%d, sg_len=%d\n",
txq->id, tx_submitted, tx_next, atomic_read(&txq->done),
tx->len, tx->sg_len);
skb_orphan(skb);
/* submit and issue descriptor */
tx->jiffies = jiffies;
tx->cookie = dmaengine_submit(dma_txd);
dma_async_issue_pending(priv->tx_chan[requested_engine]);
if (priv->n_txqs > MPODP_NOC_CHAN_COUNT)
spin_unlock_irqrestore(&priv->tx_lock[requested_engine], flags);
/* Count number of bytes on the fly for DQL */
netdev_tx_sent_queue(txq->txq, skb->len);
if (test_bit(__QUEUE_STATE_STACK_XOFF, &txq->txq->state)){
/* We reached over the limit of DQL. Try to clean some
* tx so we are rescheduled right now */
mpodp_clean_tx_unlocked(priv, txq, -1);
}
/* Increment tail pointer locally */
atomic_set(&txq->submitted, tx_next);
/* Update H2C entry offset */
tx_autoloop_next = tx_mppa_idx + 1;
if (tx_autoloop_next == txq->cached_head)
tx_autoloop_next = 0;
atomic_set(&txq->autoloop_cur, tx_autoloop_next);
skb_tx_timestamp(skb);
/* Check if there is room for another txd
* or stop the queue if there is not */
tx_next = (tx_next + 1);
if (tx_next == txq->size)
tx_next = 0;
if (tx_next == atomic_read(&txq->done)) {
if (netif_msg_tx_queued(priv))
netdev_info(netdev, "txq[%d]: ring full \n", txq->id);
netif_tx_stop_queue(txq->txq);
}
return NETDEV_TX_OK;
busy:
unmap_skb(&priv->pdev->dev, skb, tx);
return NETDEV_TX_BUSY;
addr_error:
netdev->stats.tx_dropped++;
dev_kfree_skb(skb);
/* We can't do anything, just stop the queue artificially */
netif_tx_stop_queue(txq->txq);
return NETDEV_TX_OK;
}
static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
{
struct net_device_context *net_device_ctx = netdev_priv(net);
struct hv_netvsc_packet *packet = NULL;
int ret;
unsigned int num_data_pgs;
struct rndis_message *rndis_msg;
struct rndis_packet *rndis_pkt;
u32 rndis_msg_size;
struct rndis_per_packet_info *ppi;
u32 hash;
u32 skb_length;
struct hv_page_buffer page_buf[MAX_PAGE_BUFFER_COUNT];
struct hv_page_buffer *pb = page_buf;
/* We will atmost need two pages to describe the rndis
* header. We can only transmit MAX_PAGE_BUFFER_COUNT number
* of pages in a single packet. If skb is scattered around
* more pages we try linearizing it.
*/
skb_length = skb->len;
num_data_pgs = netvsc_get_slots(skb) + 2;
if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
++net_device_ctx->eth_stats.tx_scattered;
if (skb_linearize(skb))
goto no_memory;
num_data_pgs = netvsc_get_slots(skb) + 2;
if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
++net_device_ctx->eth_stats.tx_too_big;
goto drop;
}
}
/*
* Place the rndis header in the skb head room and
* the skb->cb will be used for hv_netvsc_packet
* structure.
*/
ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
if (ret)
goto no_memory;
/* Use the skb control buffer for building up the packet */
BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
FIELD_SIZEOF(struct sk_buff, cb));
packet = (struct hv_netvsc_packet *)skb->cb;
/* TODO: This will likely evaluate to false, since RH7 and
* below kernels will set next pointer to NULL before calling
* into here. Should find another way to set this flag.
*/
packet->xmit_more = (skb->next != NULL);
packet->q_idx = skb_get_queue_mapping(skb);
packet->total_data_buflen = skb->len;
rndis_msg = (struct rndis_message *)skb->head;
memset(rndis_msg, 0, RNDIS_AND_PPI_SIZE);
packet->send_completion_ctx = packet;
/* Add the rndis header */
rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
rndis_msg->msg_len = packet->total_data_buflen;
rndis_pkt = &rndis_msg->msg.pkt;
rndis_pkt->data_offset = sizeof(struct rndis_packet);
rndis_pkt->data_len = packet->total_data_buflen;
rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet);
rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
#ifdef NOTYET
// Divergence from upstream commit:
// 307f099520b66504cf6c5638f3f404c48b9fb45b
hash = skb_get_hash_raw(skb);
#endif
hash = skb_get_hash(skb);
if (hash != 0 && net->real_num_tx_queues > 1) {
rndis_msg_size += NDIS_HASH_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
NBL_HASH_VALUE);
*(u32 *)((void *)ppi + ppi->ppi_offset) = hash;
}
if (skb_vlan_tag_present(skb)) {
struct ndis_pkt_8021q_info *vlan;
rndis_msg_size += NDIS_VLAN_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
IEEE_8021Q_INFO);
vlan = (struct ndis_pkt_8021q_info *)((void *)ppi +
ppi->ppi_offset);
vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
VLAN_PRIO_SHIFT;
}
if (skb_is_gso(skb)) {
struct ndis_tcp_lso_info *lso_info;
rndis_msg_size += NDIS_LSO_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
TCP_LARGESEND_PKTINFO);
lso_info = (struct ndis_tcp_lso_info *)((void *)ppi +
ppi->ppi_offset);
lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
if (skb->protocol == htons(ETH_P_IP)) {
lso_info->lso_v2_transmit.ip_version =
NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
ip_hdr(skb)->tot_len = 0;
ip_hdr(skb)->check = 0;
tcp_hdr(skb)->check =
~csum_tcpudp_magic(ip_hdr(skb)->saddr,
ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
} else {
lso_info->lso_v2_transmit.ip_version =
NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
ipv6_hdr(skb)->payload_len = 0;
tcp_hdr(skb)->check =
~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
&ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
}
lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb);
lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) {
struct ndis_tcp_ip_checksum_info *csum_info;
rndis_msg_size += NDIS_CSUM_PPI_SIZE;
ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
TCPIP_CHKSUM_PKTINFO);
csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi +
ppi->ppi_offset);
csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);
if (skb->protocol == htons(ETH_P_IP)) {
csum_info->transmit.is_ipv4 = 1;
if (ip_hdr(skb)->protocol == IPPROTO_TCP)
csum_info->transmit.tcp_checksum = 1;
else
csum_info->transmit.udp_checksum = 1;
} else {
csum_info->transmit.is_ipv6 = 1;
if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
csum_info->transmit.tcp_checksum = 1;
else
csum_info->transmit.udp_checksum = 1;
}
} else {
/* Can't do offload of this type of checksum */
if (skb_checksum_help(skb))
goto drop;
}
}
/* Start filling in the page buffers with the rndis hdr */
rndis_msg->msg_len += rndis_msg_size;
packet->total_data_buflen = rndis_msg->msg_len;
packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
skb, packet, &pb);
/* timestamp packet in software */
skb_tx_timestamp(skb);
ret = netvsc_send(net_device_ctx->device_ctx, packet,
rndis_msg, &pb, skb);
if (likely(ret == 0)) {
struct netvsc_stats *tx_stats = this_cpu_ptr(net_device_ctx->tx_stats);
u64_stats_update_begin(&tx_stats->syncp);
tx_stats->packets++;
tx_stats->bytes += skb_length;
u64_stats_update_end(&tx_stats->syncp);
return NETDEV_TX_OK;
}
if (ret == -EAGAIN) {
++net_device_ctx->eth_stats.tx_busy;
return NETDEV_TX_BUSY;
}
if (ret == -ENOSPC)
++net_device_ctx->eth_stats.tx_no_space;
drop:
dev_kfree_skb_any(skb);
net->stats.tx_dropped++;
return NETDEV_TX_OK;
no_memory:
++net_device_ctx->eth_stats.tx_no_memory;
goto drop;
}
/**
* stmmac_xmit:
* @skb : the socket buffer
* @dev : device pointer
* Description : Tx entry point of the driver.
*/
static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct stmmac_priv *priv = netdev_priv(dev);
unsigned int txsize = priv->dma_tx_size;
unsigned int entry;
int i, csum_insertion = 0;
int nfrags = skb_shinfo(skb)->nr_frags;
struct dma_desc *desc, *first;
if (unlikely(stmmac_tx_avail(priv) < nfrags + 1)) {
if (!netif_queue_stopped(dev)) {
netif_stop_queue(dev);
/* This is a hard error, log it. */
pr_err("%s: BUG! Tx Ring full when queue awake\n",
__func__);
}
return NETDEV_TX_BUSY;
}
entry = priv->cur_tx % txsize;
#ifdef STMMAC_XMIT_DEBUG
if ((skb->len > ETH_FRAME_LEN) || nfrags)
pr_info("stmmac xmit:\n"
"\tskb addr %p - len: %d - nopaged_len: %d\n"
"\tn_frags: %d - ip_summed: %d - %s gso\n",
skb, skb->len, skb_headlen(skb), nfrags, skb->ip_summed,
!skb_is_gso(skb) ? "isn't" : "is");
#endif
csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
desc = priv->dma_tx + entry;
first = desc;
#ifdef STMMAC_XMIT_DEBUG
if ((nfrags > 0) || (skb->len > ETH_FRAME_LEN))
pr_debug("stmmac xmit: skb len: %d, nopaged_len: %d,\n"
"\t\tn_frags: %d, ip_summed: %d\n",
skb->len, skb_headlen(skb), nfrags, skb->ip_summed);
#endif
priv->tx_skbuff[entry] = skb;
if (unlikely(skb->len >= BUF_SIZE_4KiB)) {
entry = stmmac_handle_jumbo_frames(skb, dev, csum_insertion);
desc = priv->dma_tx + entry;
} else {
unsigned int nopaged_len = skb_headlen(skb);
desc->des2 = dma_map_single(priv->device, skb->data,
nopaged_len, DMA_TO_DEVICE);
priv->hw->desc->prepare_tx_desc(desc, 1, nopaged_len,
csum_insertion);
}
for (i = 0; i < nfrags; i++) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
int len = frag->size;
entry = (++priv->cur_tx) % txsize;
desc = priv->dma_tx + entry;
TX_DBG("\t[entry %d] segment len: %d\n", entry, len);
desc->des2 = dma_map_page(priv->device, frag->page,
frag->page_offset,
len, DMA_TO_DEVICE);
priv->tx_skbuff[entry] = NULL;
priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion);
wmb();
priv->hw->desc->set_tx_owner(desc);
}
/* Interrupt on completition only for the latest segment */
priv->hw->desc->close_tx_desc(desc);
#ifdef CONFIG_STMMAC_TIMER
/* Clean IC while using timer */
if (likely(priv->tm->enable))
priv->hw->desc->clear_tx_ic(desc);
#endif
wmb();
/* To avoid raise condition */
priv->hw->desc->set_tx_owner(first);
priv->cur_tx++;
#ifdef STMMAC_XMIT_DEBUG
if (netif_msg_pktdata(priv)) {
pr_info("stmmac xmit: current=%d, dirty=%d, entry=%d, "
"first=%p, nfrags=%d\n",
(priv->cur_tx % txsize), (priv->dirty_tx % txsize),
entry, first, nfrags);
display_ring(priv->dma_tx, txsize);
pr_info(">>> frame to be transmitted: ");
print_pkt(skb->data, skb->len);
}
#endif
if (unlikely(stmmac_tx_avail(priv) <= (MAX_SKB_FRAGS + 1))) {
TX_DBG("%s: stop transmitted packets\n", __func__);
netif_stop_queue(dev);
}
dev->stats.tx_bytes += skb->len;
skb_tx_timestamp(skb);
priv->hw->dma->enable_dma_transmission(priv->ioaddr);
return NETDEV_TX_OK;
}
