/* this functions used only in no_bxm mode, * it's not implemented in netdevice.h so we have it here * based on netif_tx_lock() */ static inline int vnic_netif_tx_trylock(struct net_device *dev) { int i, cpu; spin_lock(&dev->tx_global_lock); cpu = smp_processor_id(); for (i = 0; i < dev->num_tx_queues; ++i) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i); if (__netif_tx_trylock(txq)) { set_bit(__QUEUE_STATE_FROZEN, &txq->state); __netif_tx_unlock(txq); } else { goto unlock; } } return 1; unlock: /* based on netif_tx_unlock() */ for (--i; i >= 0; --i) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i); clear_bit(__QUEUE_STATE_FROZEN, &txq->state); if (!test_bit(QUEUE_STATE_ANY_XOFF, &txq->state)) __netif_schedule(txq->qdisc); } spin_unlock(&dev->tx_global_lock); return 0; }
int mpodp_clean_tx(struct mpodp_if_priv *priv, unsigned budget) { int i; int worked = 0; for (i = 0; i < priv->n_txqs; ++i){ struct netdev_queue *txq = priv->txqs[i].txq; if (__netif_tx_trylock(txq)) { worked += mpodp_clean_tx_unlocked(priv, &priv->txqs[i], MPODP_MAX_TX_RECLAIM); __netif_tx_unlock(txq); } } return worked; }
static int lpc_eth_poll(struct napi_struct *napi, int budget) { struct netdata_local *pldat = container_of(napi, struct netdata_local, napi); struct net_device *ndev = pldat->ndev; int rx_done = 0; struct netdev_queue *txq = netdev_get_tx_queue(ndev, 0); __netif_tx_lock(txq, smp_processor_id()); __lpc_handle_xmit(ndev); __netif_tx_unlock(txq); rx_done = __lpc_handle_recv(ndev, budget); if (rx_done < budget) { napi_complete(napi); lpc_eth_enable_int(pldat->net_base); } return rx_done; }
static void ri_tasklet(unsigned long dev) { struct net_device *_dev = (struct net_device *)dev; struct ifb_private *dp = netdev_priv(_dev); struct net_device_stats *stats = &_dev->stats; struct netdev_queue *txq; struct sk_buff *skb; txq = netdev_get_tx_queue(_dev, 0); if ((skb = skb_peek(&dp->tq)) == NULL) { if (__netif_tx_trylock(txq)) { skb_queue_splice_tail_init(&dp->rq, &dp->tq); __netif_tx_unlock(txq); } else { /* reschedule */ goto resched; } } while ((skb = __skb_dequeue(&dp->tq)) != NULL) { u32 from = G_TC_FROM(skb->tc_verd); skb->tc_verd = 0; skb->tc_verd = SET_TC_NCLS(skb->tc_verd); stats->tx_packets++; stats->tx_bytes +=skb->len; rcu_read_lock(); skb->dev = dev_get_by_index_rcu(&init_net, skb->skb_iif); if (!skb->dev) { rcu_read_unlock(); dev_kfree_skb(skb); stats->tx_dropped++; if (skb_queue_len(&dp->tq) != 0) goto resched; break; } rcu_read_unlock(); skb->skb_iif = _dev->ifindex; if (from & AT_EGRESS) { dev_queue_xmit(skb); } else if (from & AT_INGRESS) { skb_pull(skb, skb->dev->hard_header_len); netif_receive_skb(skb); } else BUG(); } if (__netif_tx_trylock(txq)) { if ((skb = skb_peek(&dp->rq)) == NULL) { dp->tasklet_pending = 0; if (netif_queue_stopped(_dev)) netif_wake_queue(_dev); } else { __netif_tx_unlock(txq); goto resched; } __netif_tx_unlock(txq); } else { resched: dp->tasklet_pending = 1; tasklet_schedule(&dp->ifb_tasklet); } }
static inline void dma_xmit_clean(struct net_device *dev, END_DEVICE *ei_local) { struct netdev_queue *txq; int cpu, clean_done = 0; u32 cpu_ptr, dma_ptr, cpu_idx; #if defined (CONFIG_RAETH_BQL) u32 bytes_sent_ge1 = 0; #if defined (CONFIG_PSEUDO_SUPPORT) u32 bytes_sent_ge2 = 0; #endif #endif spin_lock(&ei_local->page_lock); cpu_ptr = sysRegRead(QTX_CRX_PTR); dma_ptr = sysRegRead(QTX_DRX_PTR); /* get current CPU TXD index */ cpu_idx = get_txd_offset(ei_local, cpu_ptr); while (cpu_ptr != dma_ptr) { struct QDMA_txdesc *txd; struct sk_buff *skb; txd = &ei_local->txd_pool[cpu_idx]; /* check TXD not owned by DMA */ if (!(ACCESS_ONCE(txd->txd_info3) & TX3_QDMA_OWN)) break; /* hold next TXD ptr */ cpu_ptr = ACCESS_ONCE(txd->txd_info2); /* release current TXD */ put_free_txd(ei_local, cpu_idx); /* get next TXD index */ cpu_idx = get_txd_offset(ei_local, cpu_ptr); /* free skb */ skb = ei_local->txd_buff[cpu_idx]; if (skb) { #if defined (CONFIG_RAETH_BQL) #if defined (CONFIG_PSEUDO_SUPPORT) if (skb->dev == ei_local->PseudoDev) bytes_sent_ge2 += skb->len; else #endif bytes_sent_ge1 += skb->len; #endif ei_local->txd_buff[cpu_idx] = NULL; dev_kfree_skb(skb); } clean_done++; /* prevent infinity loop when something wrong */ if (clean_done > (NUM_TX_DESC-4)) break; } if (clean_done) sysRegWrite(QTX_CRX_PTR, cpu_ptr); spin_unlock(&ei_local->page_lock); if (!clean_done) return; cpu = smp_processor_id(); if (netif_running(dev)) { txq = netdev_get_tx_queue(dev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge1); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #if defined (CONFIG_PSEUDO_SUPPORT) if (netif_running(ei_local->PseudoDev)) { txq = netdev_get_tx_queue(ei_local->PseudoDev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge2); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #endif }
static inline void dma_xmit_clean(struct net_device *dev, END_DEVICE *ei_local) { struct netdev_queue *txq; int cpu, clean_done = 0; u32 txd_free_idx; #if defined (CONFIG_RAETH_BQL) u32 bytes_sent_ge1 = 0; #if defined (CONFIG_PSEUDO_SUPPORT) u32 bytes_sent_ge2 = 0; #endif #endif spin_lock(&ei_local->page_lock); txd_free_idx = ei_local->txd_free_idx; while (clean_done < (NUM_TX_DESC-2)) { struct PDMA_txdesc *txd; struct sk_buff *skb; skb = ei_local->txd_buff[txd_free_idx]; if (!skb) break; txd = &ei_local->txd_ring[txd_free_idx]; /* check TXD not owned by DMA */ if (!(ACCESS_ONCE(txd->txd_info2) & TX2_DMA_DONE)) break; if (skb != (struct sk_buff *)0xFFFFFFFF) { #if defined (CONFIG_RAETH_BQL) #if defined (CONFIG_PSEUDO_SUPPORT) if (skb->dev == ei_local->PseudoDev) bytes_sent_ge2 += skb->len; else #endif bytes_sent_ge1 += skb->len; #endif dev_kfree_skb(skb); } ei_local->txd_buff[txd_free_idx] = NULL; txd_free_idx = (txd_free_idx + 1) % NUM_TX_DESC; clean_done++; } if (ei_local->txd_free_idx != txd_free_idx) ei_local->txd_free_idx = txd_free_idx; spin_unlock(&ei_local->page_lock); if (!clean_done) return; cpu = smp_processor_id(); if (netif_running(dev)) { txq = netdev_get_tx_queue(dev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge1); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #if defined (CONFIG_PSEUDO_SUPPORT) if (netif_running(ei_local->PseudoDev)) { txq = netdev_get_tx_queue(ei_local->PseudoDev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge2); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #endif }
static void ri_tasklet(unsigned long dev) { struct net_device *_dev = (struct net_device *)dev; struct ifb_private *dp = netdev_priv(_dev); struct net_device_stats *stats = &_dev->stats; struct netdev_queue *txq; struct sk_buff *skb; txq = netdev_get_tx_queue(_dev, 0); dp->st_task_enter++; if ((skb = skb_peek(&dp->tq)) == NULL) { dp->st_txq_refl_try++; if (__netif_tx_trylock(txq)) { dp->st_rxq_enter++; while ((skb = skb_dequeue(&dp->rq)) != NULL) { skb_queue_tail(&dp->tq, skb); dp->st_rx2tx_tran++; } __netif_tx_unlock(txq); } else { /* reschedule */ dp->st_rxq_notenter++; goto resched; } } while ((skb = skb_dequeue(&dp->tq)) != NULL) { u32 from = G_TC_FROM(skb->tc_verd); skb->tc_verd = 0; skb->tc_verd = SET_TC_NCLS(skb->tc_verd); stats->tx_packets++; stats->tx_bytes +=skb->len; skb->dev = dev_get_by_index(&init_net, skb->iif); if (!skb->dev) { dev_kfree_skb(skb); stats->tx_dropped++; break; } dev_put(skb->dev); skb->iif = _dev->ifindex; if (from & AT_EGRESS) { dp->st_rx_frm_egr++; dev_queue_xmit(skb); } else if (from & AT_INGRESS) { dp->st_rx_frm_ing++; skb_pull(skb, skb->dev->hard_header_len); netif_rx(skb); } else BUG(); } if (__netif_tx_trylock(txq)) { dp->st_rxq_check++; if ((skb = skb_peek(&dp->rq)) == NULL) { dp->tasklet_pending = 0; if (netif_queue_stopped(_dev)) netif_wake_queue(_dev); } else { dp->st_rxq_rsch++; __netif_tx_unlock(txq); goto resched; } __netif_tx_unlock(txq); } else { resched: dp->tasklet_pending = 1; tasklet_schedule(&dp->ifb_tasklet); } }
static void ifb_ri_tasklet(unsigned long _txp) { struct ifb_q_private *txp = (struct ifb_q_private *)_txp; struct netdev_queue *txq; struct sk_buff *skb; txq = netdev_get_tx_queue(txp->dev, txp->txqnum); skb = skb_peek(&txp->tq); if (!skb) { if (!__netif_tx_trylock(txq)) goto resched; skb_queue_splice_tail_init(&txp->rq, &txp->tq); __netif_tx_unlock(txq); } while ((skb = __skb_dequeue(&txp->tq)) != NULL) { skb->tc_redirected = 0; skb->tc_skip_classify = 1; u64_stats_update_begin(&txp->tsync); txp->tx_packets++; txp->tx_bytes += skb->len; u64_stats_update_end(&txp->tsync); rcu_read_lock(); skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif); if (!skb->dev) { rcu_read_unlock(); dev_kfree_skb(skb); txp->dev->stats.tx_dropped++; if (skb_queue_len(&txp->tq) != 0) goto resched; break; } rcu_read_unlock(); skb->skb_iif = txp->dev->ifindex; if (!skb->tc_from_ingress) { dev_queue_xmit(skb); } else { skb_pull_rcsum(skb, skb->mac_len); netif_receive_skb(skb); } } if (__netif_tx_trylock(txq)) { skb = skb_peek(&txp->rq); if (!skb) { txp->tasklet_pending = 0; if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); } else { __netif_tx_unlock(txq); goto resched; } __netif_tx_unlock(txq); } else { resched: txp->tasklet_pending = 1; tasklet_schedule(&txp->ifb_tasklet); } }