/*
* Add buffer into wmm tx queue and queue work to transmit it.
*/
int mwifiex_queue_tx_pkt(struct mwifiex_private *priv, struct sk_buff *skb)
{
struct netdev_queue *txq;
int index = mwifiex_1d_to_wmm_queue[skb->priority];
if (atomic_inc_return(&priv->wmm_tx_pending[index]) >= MAX_TX_PENDING) {
txq = netdev_get_tx_queue(priv->netdev, index);
if (!netif_tx_queue_stopped(txq)) {
netif_tx_stop_queue(txq);
mwifiex_dbg(priv->adapter, DATA,
"stop queue: %d\n", index);
}
}
if (mwifiex_bypass_tx_queue(priv, skb)) {
atomic_inc(&priv->adapter->tx_pending);
atomic_inc(&priv->adapter->bypass_tx_pending);
mwifiex_wmm_add_buf_bypass_txqueue(priv, skb);
} else {
atomic_inc(&priv->adapter->tx_pending);
mwifiex_wmm_add_buf_txqueue(priv, skb);
}
mwifiex_queue_main_work(priv->adapter);
return 0;
}
static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct ifb_dev_private *dp = netdev_priv(dev);
struct ifb_q_private *txp = dp->tx_private + skb_get_queue_mapping(skb);
u64_stats_update_begin(&txp->rsync);
txp->rx_packets++;
txp->rx_bytes += skb->len;
u64_stats_update_end(&txp->rsync);
if (!skb->tc_redirected || !skb->skb_iif) {
dev_kfree_skb(skb);
dev->stats.rx_dropped++;
return NETDEV_TX_OK;
}
if (skb_queue_len(&txp->rq) >= dev->tx_queue_len)
netif_tx_stop_queue(netdev_get_tx_queue(dev, txp->txqnum));
__skb_queue_tail(&txp->rq, skb);
if (!txp->tasklet_pending) {
txp->tasklet_pending = 1;
tasklet_schedule(&txp->ifb_tasklet);
}
return NETDEV_TX_OK;
}
void netif_tx_stop_all_queues(struct net_device *dev) {
uint32_t i;
for(i = 0; i < dev->num_tx_queues; i++) {
struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
txq->state = 0;
netif_tx_stop_queue(txq);
}
}
static inline void xenvif_stop_queue(struct xenvif_queue *queue)
{
struct net_device *dev = queue->vif->dev;
if (!queue->vif->can_queue)
return;
netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));
}
/**
* nfp_net_tx_ring_stop() - stop tx ring
* @nd_q: netdev queue
* @tx_ring: driver tx queue structure
*
* Safely stop TX ring. Remember that while we are running .start_xmit()
* someone else may be cleaning the TX ring completions so we need to be
* extra careful here.
*/
static void nfp_net_tx_ring_stop(struct netdev_queue *nd_q,
struct nfp_net_tx_ring *tx_ring)
{
netif_tx_stop_queue(nd_q);
/* We can race with the TX completion out of NAPI so recheck */
smp_mb();
if (unlikely(nfp_net_tx_ring_should_wake(tx_ring)))
netif_tx_start_queue(nd_q);
}
/*
* This function stops all queues in net_device
*/
void mwifiex_stop_net_dev_queue(struct net_device *netdev,
struct mwifiex_adapter *adapter)
{
unsigned long dev_queue_flags;
unsigned int i;
spin_lock_irqsave(&adapter->queue_lock, dev_queue_flags);
for (i = 0; i < netdev->num_tx_queues; i++) {
struct netdev_queue *txq = netdev_get_tx_queue(netdev, i);
if (!netif_tx_queue_stopped(txq))
netif_tx_stop_queue(txq);
}
spin_unlock_irqrestore(&adapter->queue_lock, dev_queue_flags);
}
static void efx_tx_maybe_stop_queue(struct efx_tx_queue *txq1)
{
/* We need to consider both queues that the net core sees as one */
struct efx_tx_queue *txq2 = efx_tx_queue_partner(txq1);
struct efx_nic *efx = txq1->efx;
unsigned int fill_level;
fill_level = max(txq1->insert_count - txq1->old_read_count,
txq2->insert_count - txq2->old_read_count);
if (likely(fill_level < efx->txq_stop_thresh))
return;
/* We used the stale old_read_count above, which gives us a
* pessimistic estimate of the fill level (which may even
* validly be >= efx->txq_entries). Now try again using
* read_count (more likely to be a cache miss).
*
* If we read read_count and then conditionally stop the
* queue, it is possible for the completion path to race with
* us and complete all outstanding descriptors in the middle,
* after which there will be no more completions to wake it.
* Therefore we stop the queue first, then read read_count
* (with a memory barrier to ensure the ordering), then
* restart the queue if the fill level turns out to be low
* enough.
*/
netif_tx_stop_queue(txq1->core_txq);
smp_mb();
txq1->old_read_count = ACCESS_ONCE(txq1->read_count);
txq2->old_read_count = ACCESS_ONCE(txq2->read_count);
fill_level = max(txq1->insert_count - txq1->old_read_count,
txq2->insert_count - txq2->old_read_count);
EFX_BUG_ON_PARANOID(fill_level >= efx->txq_entries);
if (likely(fill_level < efx->txq_stop_thresh)) {
smp_mb();
if (likely(!efx->loopback_selftest))
netif_tx_start_queue(txq1->core_txq);
}
}
/*
* Add buffer into wmm tx queue and queue work to transmit it.
*/
int mwifiex_queue_tx_pkt(struct mwifiex_private *priv, struct sk_buff *skb)
{
struct netdev_queue *txq;
int index = mwifiex_1d_to_wmm_queue[skb->priority];
if (atomic_inc_return(&priv->wmm_tx_pending[index]) >= MAX_TX_PENDING) {
txq = netdev_get_tx_queue(priv->netdev, index);
if (!netif_tx_queue_stopped(txq)) {
netif_tx_stop_queue(txq);
dev_dbg(priv->adapter->dev, "stop queue: %d\n", index);
}
}
atomic_inc(&priv->adapter->tx_pending);
mwifiex_wmm_add_buf_txqueue(priv, skb);
if (priv->adapter->scan_delay_cnt)
atomic_set(&priv->adapter->is_tx_received, true);
queue_work(priv->adapter->workqueue, &priv->adapter->main_work);
return 0;
}
static netdev_tx_t
fjes_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
{
struct fjes_adapter *adapter = netdev_priv(netdev);
struct fjes_hw *hw = &adapter->hw;
int max_epid, my_epid, dest_epid;
enum ep_partner_status pstatus;
struct netdev_queue *cur_queue;
char shortpkt[VLAN_ETH_HLEN];
bool is_multi, vlan;
struct ethhdr *eth;
u16 queue_no = 0;
u16 vlan_id = 0;
netdev_tx_t ret;
char *data;
int len;
ret = NETDEV_TX_OK;
is_multi = false;
cur_queue = netdev_get_tx_queue(netdev, queue_no);
eth = (struct ethhdr *)skb->data;
my_epid = hw->my_epid;
vlan = (vlan_get_tag(skb, &vlan_id) == 0) ? true : false;
data = skb->data;
len = skb->len;
if (is_multicast_ether_addr(eth->h_dest)) {
dest_epid = 0;
max_epid = hw->max_epid;
is_multi = true;
} else if (is_local_ether_addr(eth->h_dest)) {
dest_epid = eth->h_dest[ETH_ALEN - 1];
max_epid = dest_epid + 1;
if ((eth->h_dest[0] == 0x02) &&
(0x00 == (eth->h_dest[1] | eth->h_dest[2] |
eth->h_dest[3] | eth->h_dest[4])) &&
(dest_epid < hw->max_epid)) {
;
} else {
dest_epid = 0;
max_epid = 0;
ret = NETDEV_TX_OK;
adapter->stats64.tx_packets += 1;
hw->ep_shm_info[my_epid].net_stats.tx_packets += 1;
adapter->stats64.tx_bytes += len;
hw->ep_shm_info[my_epid].net_stats.tx_bytes += len;
}
} else {
dest_epid = 0;
max_epid = 0;
ret = NETDEV_TX_OK;
adapter->stats64.tx_packets += 1;
hw->ep_shm_info[my_epid].net_stats.tx_packets += 1;
adapter->stats64.tx_bytes += len;
hw->ep_shm_info[my_epid].net_stats.tx_bytes += len;
}
for (; dest_epid < max_epid; dest_epid++) {
if (my_epid == dest_epid)
continue;
pstatus = fjes_hw_get_partner_ep_status(hw, dest_epid);
if (pstatus != EP_PARTNER_SHARED) {
if (!is_multi)
hw->ep_shm_info[dest_epid].ep_stats
.tx_dropped_not_shared += 1;
ret = NETDEV_TX_OK;
} else if (!fjes_hw_check_epbuf_version(
&adapter->hw.ep_shm_info[dest_epid].rx, 0)) {
/* version is NOT 0 */
adapter->stats64.tx_carrier_errors += 1;
hw->ep_shm_info[dest_epid].net_stats
.tx_carrier_errors += 1;
hw->ep_shm_info[dest_epid].ep_stats
.tx_dropped_ver_mismatch += 1;
ret = NETDEV_TX_OK;
} else if (!fjes_hw_check_mtu(
&adapter->hw.ep_shm_info[dest_epid].rx,
netdev->mtu)) {
adapter->stats64.tx_dropped += 1;
hw->ep_shm_info[dest_epid].net_stats.tx_dropped += 1;
adapter->stats64.tx_errors += 1;
hw->ep_shm_info[dest_epid].net_stats.tx_errors += 1;
hw->ep_shm_info[dest_epid].ep_stats
.tx_dropped_buf_size_mismatch += 1;
ret = NETDEV_TX_OK;
} else if (vlan &&
!fjes_hw_check_vlan_id(
&adapter->hw.ep_shm_info[dest_epid].rx,
vlan_id)) {
hw->ep_shm_info[dest_epid].ep_stats
.tx_dropped_vlanid_mismatch += 1;
ret = NETDEV_TX_OK;
} else {
if (len < VLAN_ETH_HLEN) {
memset(shortpkt, 0, VLAN_ETH_HLEN);
memcpy(shortpkt, skb->data, skb->len);
len = VLAN_ETH_HLEN;
data = shortpkt;
}
if (adapter->tx_retry_count == 0) {
adapter->tx_start_jiffies = jiffies;
adapter->tx_retry_count = 1;
} else {
adapter->tx_retry_count++;
}
if (fjes_tx_send(adapter, dest_epid, data, len)) {
if (is_multi) {
ret = NETDEV_TX_OK;
} else if (
((long)jiffies -
(long)adapter->tx_start_jiffies) >=
FJES_TX_RETRY_TIMEOUT) {
adapter->stats64.tx_fifo_errors += 1;
hw->ep_shm_info[dest_epid].net_stats
.tx_fifo_errors += 1;
adapter->stats64.tx_errors += 1;
hw->ep_shm_info[dest_epid].net_stats
.tx_errors += 1;
ret = NETDEV_TX_OK;
} else {
netif_trans_update(netdev);
hw->ep_shm_info[dest_epid].ep_stats
.tx_buffer_full += 1;
netif_tx_stop_queue(cur_queue);
if (!work_pending(&adapter->tx_stall_task))
queue_work(adapter->txrx_wq,
&adapter->tx_stall_task);
ret = NETDEV_TX_BUSY;
}
} else {
if (!is_multi) {
adapter->stats64.tx_packets += 1;
hw->ep_shm_info[dest_epid].net_stats
.tx_packets += 1;
adapter->stats64.tx_bytes += len;
hw->ep_shm_info[dest_epid].net_stats
.tx_bytes += len;
}
adapter->tx_retry_count = 0;
ret = NETDEV_TX_OK;
}
}
}
if (ret == NETDEV_TX_OK) {
dev_kfree_skb(skb);
if (is_multi) {
adapter->stats64.tx_packets += 1;
hw->ep_shm_info[my_epid].net_stats.tx_packets += 1;
adapter->stats64.tx_bytes += 1;
hw->ep_shm_info[my_epid].net_stats.tx_bytes += len;
}
}
return ret;
}
netdev_tx_t mpodp_start_xmit(struct sk_buff *skb,
struct net_device *netdev)
{
struct mpodp_if_priv *priv = netdev_priv(netdev);
struct mpodp_tx *tx;
struct dma_async_tx_descriptor *dma_txd;
struct mpodp_cache_entry *entry;
int ret;
uint8_t fifo_mode;
int16_t requested_engine;
struct mpodp_pkt_hdr *hdr;
uint32_t tx_autoloop_next;
uint32_t tx_submitted, tx_next, tx_done;
uint32_t tx_mppa_idx;
int qidx;
unsigned long flags = 0;
struct mpodp_txq *txq;
/* Fetch HW queue selected by the kernel */
qidx = skb_get_queue_mapping(skb);
txq = &priv->txqs[qidx];
if (atomic_read(&priv->reset) == 1) {
mpodp_clean_tx_unlocked(priv, txq, -1);
goto addr_error;
}
tx_submitted = atomic_read(&txq->submitted);
/* Compute txd id */
tx_next = (tx_submitted + 1);
if (tx_next == txq->size)
tx_next = 0;
/* MPPA H2C Entry to use */
tx_mppa_idx = atomic_read(&txq->autoloop_cur);
tx_done = atomic_read(&txq->done);
if (tx_done != tx_submitted &&
((txq->ring[tx_done].jiffies + msecs_to_jiffies(5) >= jiffies) ||
(tx_submitted < tx_done && tx_submitted + txq->size - tx_done >= TX_POLL_THRESHOLD) ||
(tx_submitted >= tx_done && tx_submitted - tx_done >= TX_POLL_THRESHOLD))) {
mpodp_clean_tx_unlocked(priv, txq, -1);
}
/* Check if there are txd available */
if (tx_next == atomic_read(&txq->done)) {
/* Ring is full */
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d]: ring full \n", txq->id);
netif_tx_stop_queue(txq->txq);
return NETDEV_TX_BUSY;
}
tx = &(txq->ring[tx_submitted]);
entry = &(txq->cache[tx_mppa_idx]);
/* take the time */
mppa_pcie_time_get(priv->tx_time, &tx->time);
/* configure channel */
tx->dst_addr = entry->addr;
/* Check the provided address */
ret =
mppa_pcie_dma_check_addr(priv->pdata, tx->dst_addr, &fifo_mode,
&requested_engine);
if (ret) {
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: invalid send address %llx\n",
txq->id, tx_submitted, tx->dst_addr);
goto addr_error;
}
if (!fifo_mode) {
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: %llx is not a PCI2Noc addres\n",
txq->id, tx_submitted, tx->dst_addr);
goto addr_error;
}
if (requested_engine >= MPODP_NOC_CHAN_COUNT) {
if (netif_msg_tx_err(priv))
netdev_err(netdev,
"txq[%d] tx[%d]: address %llx using NoC engine out of range (%d >= %d)\n",
txq->id, tx_submitted, tx->dst_addr,
requested_engine, MPODP_NOC_CHAN_COUNT);
goto addr_error;
}
tx->chanidx = requested_engine;
/* The packet needs a header to determine size,timestamp, etc.
* Add it */
if (skb_headroom(skb) < sizeof(struct mpodp_pkt_hdr)) {
struct sk_buff *skb_new;
skb_new =
skb_realloc_headroom(skb, sizeof(struct mpodp_pkt_hdr));
if (!skb_new) {
netdev->stats.tx_errors++;
kfree_skb(skb);
return NETDEV_TX_OK;
}
kfree_skb(skb);
skb = skb_new;
}
hdr = (struct mpodp_pkt_hdr *)
skb_push(skb, sizeof(struct mpodp_pkt_hdr));
hdr->timestamp = priv->packet_id;
hdr->info._.pkt_id = priv->packet_id;
hdr->info.dword = 0ULL;
hdr->info._.pkt_size = skb->len; /* Also count the header size */
hdr->info._.pkt_id = priv->packet_id;
priv->packet_id++;
/* save skb to free it later */
tx->skb = skb;
tx->len = skb->len;
/* prepare sg */
if (map_skb(&priv->pdev->dev, skb, tx)){
if (netif_msg_tx_err(priv))
netdev_err(netdev, "tx %d: failed to map skb to dma\n",
tx_submitted);
goto busy;
}
if (priv->n_txqs > MPODP_NOC_CHAN_COUNT)
spin_lock_irqsave(&priv->tx_lock[requested_engine], flags);
/* Prepare slave args */
priv->tx_config[requested_engine].cfg.dst_addr = tx->dst_addr;
priv->tx_config[requested_engine].requested_engine = requested_engine;
/* FIFO mode, direction, latency were filled at setup */
if (dmaengine_slave_config(priv->tx_chan[requested_engine],
&priv->tx_config[requested_engine].cfg)) {
/* board has reset, wait for reset of netdev */
netif_tx_stop_queue(txq->txq);
netif_carrier_off(netdev);
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: cannot configure channel\n",
txq->id, tx_submitted);
goto busy;
}
/* get transfer descriptor */
dma_txd =
dmaengine_prep_slave_sg(priv->tx_chan[requested_engine], tx->sg,
tx->sg_len, DMA_MEM_TO_DEV, 0);
if (dma_txd == NULL) {
/* dmaengine_prep_slave_sg failed, retry */
if (netif_msg_tx_err(priv))
netdev_err(netdev, "txq[%d] tx[%d]: cannot get dma descriptor\n",
txq->id, tx_submitted);
goto busy;
}
if (netif_msg_tx_queued(priv))
netdev_info(netdev,
"txq[%d] tx[%d]: transfer start (submitted: %d done: %d) len=%d, sg_len=%d\n",
txq->id, tx_submitted, tx_next, atomic_read(&txq->done),
tx->len, tx->sg_len);
skb_orphan(skb);
/* submit and issue descriptor */
tx->jiffies = jiffies;
tx->cookie = dmaengine_submit(dma_txd);
dma_async_issue_pending(priv->tx_chan[requested_engine]);
if (priv->n_txqs > MPODP_NOC_CHAN_COUNT)
spin_unlock_irqrestore(&priv->tx_lock[requested_engine], flags);
/* Count number of bytes on the fly for DQL */
netdev_tx_sent_queue(txq->txq, skb->len);
if (test_bit(__QUEUE_STATE_STACK_XOFF, &txq->txq->state)){
/* We reached over the limit of DQL. Try to clean some
* tx so we are rescheduled right now */
mpodp_clean_tx_unlocked(priv, txq, -1);
}
/* Increment tail pointer locally */
atomic_set(&txq->submitted, tx_next);
/* Update H2C entry offset */
tx_autoloop_next = tx_mppa_idx + 1;
if (tx_autoloop_next == txq->cached_head)
tx_autoloop_next = 0;
atomic_set(&txq->autoloop_cur, tx_autoloop_next);
skb_tx_timestamp(skb);
/* Check if there is room for another txd
* or stop the queue if there is not */
tx_next = (tx_next + 1);
if (tx_next == txq->size)
tx_next = 0;
if (tx_next == atomic_read(&txq->done)) {
if (netif_msg_tx_queued(priv))
netdev_info(netdev, "txq[%d]: ring full \n", txq->id);
netif_tx_stop_queue(txq->txq);
}
return NETDEV_TX_OK;
busy:
unmap_skb(&priv->pdev->dev, skb, tx);
return NETDEV_TX_BUSY;
addr_error:
netdev->stats.tx_dropped++;
dev_kfree_skb(skb);
/* We can't do anything, just stop the queue artificially */
netif_tx_stop_queue(txq->txq);
return NETDEV_TX_OK;
}
/**
* nfp_net_tx() - Main transmit entry point
* @skb: SKB to transmit
* @netdev: netdev structure
*
* Return: NETDEV_TX_OK on success.
*/
static int nfp_net_tx(struct sk_buff *skb, struct net_device *netdev)
{
struct nfp_net *nn = netdev_priv(netdev);
const struct skb_frag_struct *frag;
struct nfp_net_r_vector *r_vec;
struct nfp_net_tx_desc *txd, txdg;
struct nfp_net_tx_buf *txbuf;
struct nfp_net_tx_ring *tx_ring;
struct netdev_queue *nd_q;
dma_addr_t dma_addr;
unsigned int fsize;
int f, nr_frags;
int wr_idx;
u16 qidx;
qidx = skb_get_queue_mapping(skb);
tx_ring = &nn->tx_rings[qidx];
r_vec = tx_ring->r_vec;
nd_q = netdev_get_tx_queue(nn->netdev, qidx);
nr_frags = skb_shinfo(skb)->nr_frags;
if (unlikely(nfp_net_tx_full(tx_ring, nr_frags + 1))) {
nn_warn_ratelimit(nn, "TX ring %d busy. wrp=%u rdp=%u\n",
qidx, tx_ring->wr_p, tx_ring->rd_p);
netif_tx_stop_queue(nd_q);
u64_stats_update_begin(&r_vec->tx_sync);
r_vec->tx_busy++;
u64_stats_update_end(&r_vec->tx_sync);
return NETDEV_TX_BUSY;
}
/* Start with the head skbuf */
dma_addr = dma_map_single(&nn->pdev->dev, skb->data, skb_headlen(skb),
DMA_TO_DEVICE);
if (dma_mapping_error(&nn->pdev->dev, dma_addr))
goto err_free;
wr_idx = tx_ring->wr_p % tx_ring->cnt;
/* Stash the soft descriptor of the head then initialize it */
txbuf = &tx_ring->txbufs[wr_idx];
txbuf->skb = skb;
txbuf->dma_addr = dma_addr;
txbuf->fidx = -1;
txbuf->pkt_cnt = 1;
txbuf->real_len = skb->len;
/* Build TX descriptor */
txd = &tx_ring->txds[wr_idx];
txd->offset_eop = (nr_frags == 0) ? PCIE_DESC_TX_EOP : 0;
txd->dma_len = cpu_to_le16(skb_headlen(skb));
nfp_desc_set_dma_addr(txd, dma_addr);
txd->data_len = cpu_to_le16(skb->len);
txd->flags = 0;
txd->mss = 0;
txd->l4_offset = 0;
nfp_net_tx_tso(nn, r_vec, txbuf, txd, skb);
nfp_net_tx_csum(nn, r_vec, txbuf, txd, skb);
if (skb_vlan_tag_present(skb) && nn->ctrl & NFP_NET_CFG_CTRL_TXVLAN) {
txd->flags |= PCIE_DESC_TX_VLAN;
txd->vlan = cpu_to_le16(skb_vlan_tag_get(skb));
}
/* Gather DMA */
if (nr_frags > 0) {
/* all descs must match except for in addr, length and eop */
txdg = *txd;
for (f = 0; f < nr_frags; f++) {
frag = &skb_shinfo(skb)->frags[f];
fsize = skb_frag_size(frag);
dma_addr = skb_frag_dma_map(&nn->pdev->dev, frag, 0,
fsize, DMA_TO_DEVICE);
if (dma_mapping_error(&nn->pdev->dev, dma_addr))
goto err_unmap;
wr_idx = (wr_idx + 1) % tx_ring->cnt;
tx_ring->txbufs[wr_idx].skb = skb;
tx_ring->txbufs[wr_idx].dma_addr = dma_addr;
tx_ring->txbufs[wr_idx].fidx = f;
txd = &tx_ring->txds[wr_idx];
*txd = txdg;
txd->dma_len = cpu_to_le16(fsize);
nfp_desc_set_dma_addr(txd, dma_addr);
txd->offset_eop =
(f == nr_frags - 1) ? PCIE_DESC_TX_EOP : 0;
}
u64_stats_update_begin(&r_vec->tx_sync);
r_vec->tx_gather++;
u64_stats_update_end(&r_vec->tx_sync);
}
netdev_tx_sent_queue(nd_q, txbuf->real_len);
tx_ring->wr_p += nr_frags + 1;
if (nfp_net_tx_ring_should_stop(tx_ring))
nfp_net_tx_ring_stop(nd_q, tx_ring);
tx_ring->wr_ptr_add += nr_frags + 1;
if (!skb->xmit_more || netif_xmit_stopped(nd_q)) {
/* force memory write before we let HW know */
wmb();
nfp_qcp_wr_ptr_add(tx_ring->qcp_q, tx_ring->wr_ptr_add);
tx_ring->wr_ptr_add = 0;
}
skb_tx_timestamp(skb);
return NETDEV_TX_OK;
err_unmap:
--f;
while (f >= 0) {
frag = &skb_shinfo(skb)->frags[f];
dma_unmap_page(&nn->pdev->dev,
tx_ring->txbufs[wr_idx].dma_addr,
skb_frag_size(frag), DMA_TO_DEVICE);
tx_ring->txbufs[wr_idx].skb = NULL;
tx_ring->txbufs[wr_idx].dma_addr = 0;
tx_ring->txbufs[wr_idx].fidx = -2;
wr_idx = wr_idx - 1;
if (wr_idx < 0)
wr_idx += tx_ring->cnt;
}
dma_unmap_single(&nn->pdev->dev, tx_ring->txbufs[wr_idx].dma_addr,
skb_headlen(skb), DMA_TO_DEVICE);
tx_ring->txbufs[wr_idx].skb = NULL;
tx_ring->txbufs[wr_idx].dma_addr = 0;
tx_ring->txbufs[wr_idx].fidx = -2;
err_free:
nn_warn_ratelimit(nn, "Failed to map DMA TX buffer\n");
u64_stats_update_begin(&r_vec->tx_sync);
r_vec->tx_errors++;
u64_stats_update_end(&r_vec->tx_sync);
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
int netvsc_send(struct hv_device *device,
struct hv_netvsc_packet *packet, bool kick_q)
{
struct netvsc_device *net_device;
int ret = 0;
struct nvsp_message sendMessage;
struct net_device *ndev;
struct vmbus_channel *out_channel = NULL;
u64 req_id;
unsigned int section_index = NETVSC_INVALID_INDEX;
u32 msg_size = 0;
u16 q_idx = packet->q_idx;
u32 vmbus_flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
net_device = get_outbound_net_device(device);
if (!net_device)
return -ENODEV;
ndev = net_device->ndev;
sendMessage.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
if (packet->is_data_pkt) {
/* 0 is RMC_DATA; */
sendMessage.msg.v1_msg.send_rndis_pkt.channel_type = 0;
} else {
/* 1 is RMC_CONTROL; */
sendMessage.msg.v1_msg.send_rndis_pkt.channel_type = 1;
}
/* Attempt to send via sendbuf */
if (packet->total_data_buflen < net_device->send_section_size) {
section_index = netvsc_get_next_send_section(net_device);
if (section_index != NETVSC_INVALID_INDEX) {
msg_size = netvsc_copy_to_send_buf(net_device,
section_index,
packet);
packet->page_buf_cnt = 0;
}
}
packet->send_buf_index = section_index;
sendMessage.msg.v1_msg.send_rndis_pkt.send_buf_section_index =
section_index;
sendMessage.msg.v1_msg.send_rndis_pkt.send_buf_section_size = msg_size;
if (packet->send_completion)
req_id = (ulong)packet;
else
req_id = 0;
out_channel = net_device->chn_table[packet->q_idx];
if (out_channel == NULL)
out_channel = device->channel;
packet->channel = out_channel;
if (out_channel->rescind)
return -ENODEV;
if (packet->page_buf_cnt) {
ret = vmbus_sendpacket_pagebuffer_ctl(out_channel,
packet->page_buf,
packet->page_buf_cnt,
&sendMessage,
sizeof(struct nvsp_message),
req_id,
vmbus_flags,
kick_q);
} else {
ret = vmbus_sendpacket_ctl(out_channel, &sendMessage,
sizeof(struct nvsp_message),
req_id,
VM_PKT_DATA_INBAND,
vmbus_flags,
kick_q);
}
if (ret == 0) {
atomic_inc(&net_device->num_outstanding_sends);
atomic_inc(&net_device->queue_sends[q_idx]);
if (hv_ringbuf_avail_percent(&out_channel->outbound) <
RING_AVAIL_PERCENT_LOWATER) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->
queue_sends[q_idx]) < 1)
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
}
} else if (ret == -EAGAIN) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->queue_sends[q_idx]) < 1) {
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
ret = -ENOSPC;
}
} else {
netdev_err(ndev, "Unable to send packet %p ret %d\n",
packet, ret);
}
if (ret != 0 && section_index != NETVSC_INVALID_INDEX)
netvsc_free_send_slot(net_device, section_index);
return ret;
}
static inline int
dma_xmit(struct sk_buff *skb, struct net_device *dev, END_DEVICE *ei_local, int gmac_no)
{
struct netdev_queue *txq;
dma_addr_t frag_addr;
u32 frag_size, nr_desc;
u32 txd_info3, txd_info4;
#if defined (CONFIG_RAETH_SG_DMA_TX)
u32 i, nr_frags;
const skb_frag_t *tx_frag;
const struct skb_shared_info *shinfo;
#else
#define nr_frags 0
#endif
#if defined (CONFIG_RA_HW_NAT) || defined (CONFIG_RA_HW_NAT_MODULE)
if (ra_sw_nat_hook_tx != NULL) {
#if defined (CONFIG_RA_HW_NAT_WIFI) || defined (CONFIG_RA_HW_NAT_PCI)
if (IS_DPORT_PPE_VALID(skb))
gmac_no = PSE_PORT_PPE;
else
#endif
if (ra_sw_nat_hook_tx(skb, gmac_no) == 0) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
}
#endif
txd_info3 = TX3_QDMA_SWC;
if (gmac_no != PSE_PORT_PPE) {
u32 QID = M2Q_table[(skb->mark & 0x3f)];
if (QID < 8 && M2Q_wan_lan) {
#if defined (CONFIG_PSEUDO_SUPPORT)
if (gmac_no == PSE_PORT_GMAC2)
QID += 8;
#elif defined (CONFIG_RAETH_HW_VLAN_TX)
if ((skb_vlan_tag_get(skb) & VLAN_VID_MASK) > 1)
QID += 8;
#endif
}
txd_info3 |= TX3_QDMA_QID(QID);
}
txd_info4 = TX4_DMA_FPORT(gmac_no);
#if defined (CONFIG_RAETH_CHECKSUM_OFFLOAD)
if (skb->ip_summed == CHECKSUM_PARTIAL)
txd_info4 |= TX4_DMA_TUI_CO(7);
#endif
#if defined (CONFIG_RAETH_HW_VLAN_TX)
if (skb_vlan_tag_present(skb))
txd_info4 |= (0x10000 | skb_vlan_tag_get(skb));
#endif
#if defined (CONFIG_RAETH_SG_DMA_TX)
shinfo = skb_shinfo(skb);
#endif
#if defined (CONFIG_RAETH_TSO)
/* fill MSS info in tcp checksum field */
if (shinfo->gso_size) {
u32 hdr_len;
if (!(shinfo->gso_type & (SKB_GSO_TCPV4|SKB_GSO_TCPV6))) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
if (skb_header_cloned(skb)) {
if (pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
}
hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb));
if (hdr_len >= skb->len) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
tcp_hdr(skb)->check = htons(shinfo->gso_size);
txd_info4 |= TX4_DMA_TSO;
}
#endif
nr_desc = DIV_ROUND_UP(skb_headlen(skb), TXD_MAX_SEG_SIZE);
#if defined (CONFIG_RAETH_SG_DMA_TX)
nr_frags = (u32)shinfo->nr_frags;
for (i = 0; i < nr_frags; i++) {
tx_frag = &shinfo->frags[i];
nr_desc += DIV_ROUND_UP(skb_frag_size(tx_frag), TXD_MAX_SEG_SIZE);
}
#endif
txq = netdev_get_tx_queue(dev, 0);
/* flush main skb part before spin_lock() */
frag_size = (u32)skb_headlen(skb);
frag_addr = dma_map_single(NULL, skb->data, frag_size, DMA_TO_DEVICE);
/* protect TX ring access (from eth2/eth3 queues) */
spin_lock(&ei_local->page_lock);
/* check nr_desc+2 free descriptors (2 need to prevent head/tail overlap) */
if (ei_local->txd_pool_free_num < (nr_desc+2)) {
spin_unlock(&ei_local->page_lock);
netif_tx_stop_queue(txq);
#if defined (CONFIG_RAETH_DEBUG)
if (net_ratelimit())
printk("%s: QDMA TX pool is run out! (GMAC: %d)\n", RAETH_DEV_NAME, gmac_no);
#endif
return NETDEV_TX_BUSY;
}
qdma_write_skb_fragment(ei_local, frag_addr, frag_size,
txd_info3, txd_info4, skb, nr_frags == 0);
#if defined (CONFIG_RAETH_SG_DMA_TX)
for (i = 0; i < nr_frags; i++) {
tx_frag = &shinfo->frags[i];
frag_size = skb_frag_size(tx_frag);
frag_addr = skb_frag_dma_map(NULL, tx_frag, 0, frag_size, DMA_TO_DEVICE);
qdma_write_skb_fragment(ei_local, frag_addr, frag_size,
txd_info3, txd_info4, skb, i == nr_frags - 1);
}
#endif
#if defined (CONFIG_RAETH_BQL)
netdev_tx_sent_queue(txq, skb->len);
#endif
#if !defined (CONFIG_RAETH_BQL) || !defined (CONFIG_SMP)
/* smp_mb() already inlined in netdev_tx_sent_queue */
wmb();
#endif
/* kick the QDMA TX */
sysRegWrite(QTX_CTX_PTR, (u32)get_txd_ptr_phy(ei_local, ei_local->txd_last_idx));
spin_unlock(&ei_local->page_lock);
return NETDEV_TX_OK;
}
static inline int
dma_xmit(struct sk_buff* skb, struct net_device *dev, END_DEVICE *ei_local, int gmac_no)
{
struct netdev_queue *txq;
dma_addr_t frag_addr;
u32 frag_size, nr_desc;
u32 next_idx, desc_odd = 0;
u32 txd_info2 = 0, txd_info4;
#if defined (CONFIG_RAETH_SG_DMA_TX)
u32 i, nr_frags;
const skb_frag_t *tx_frag;
const struct skb_shared_info *shinfo;
#else
#define nr_frags 0
#endif
#if defined (CONFIG_RA_HW_NAT) || defined (CONFIG_RA_HW_NAT_MODULE)
if (ra_sw_nat_hook_tx != NULL) {
#if defined (CONFIG_RA_HW_NAT_WIFI) || defined (CONFIG_RA_HW_NAT_PCI)
if (IS_DPORT_PPE_VALID(skb))
gmac_no = PSE_PORT_PPE;
else
#endif
if (ra_sw_nat_hook_tx(skb, gmac_no) == 0) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
}
#endif
#if !defined (RAETH_HW_PADPKT)
if (skb->len < ei_local->min_pkt_len) {
if (skb_padto(skb, ei_local->min_pkt_len)) {
#if defined (CONFIG_RAETH_DEBUG)
if (net_ratelimit())
printk(KERN_ERR "%s: skb_padto failed\n", RAETH_DEV_NAME);
#endif
return NETDEV_TX_OK;
}
skb_put(skb, ei_local->min_pkt_len - skb->len);
}
#endif
#if defined (CONFIG_RALINK_MT7620)
if (gmac_no == PSE_PORT_PPE)
txd_info4 = TX4_DMA_FP_BMAP(0x80); /* P7 */
else
#if defined (CONFIG_RAETH_HAS_PORT5) && !defined (CONFIG_RAETH_HAS_PORT4) && !defined (CONFIG_RAETH_ESW)
txd_info4 = TX4_DMA_FP_BMAP(0x20); /* P5 */
#elif defined (CONFIG_RAETH_HAS_PORT4) && !defined (CONFIG_RAETH_HAS_PORT5) && !defined (CONFIG_RAETH_ESW)
txd_info4 = TX4_DMA_FP_BMAP(0x10); /* P4 */
#else
txd_info4 = 0; /* routing by DA */
#endif
#elif defined (CONFIG_RALINK_MT7621)
txd_info4 = TX4_DMA_FPORT(gmac_no);
#else
txd_info4 = (TX4_DMA_QN(3) | TX4_DMA_PN(gmac_no));
#endif
#if defined (CONFIG_RAETH_CHECKSUM_OFFLOAD) && !defined (RAETH_SDMA)
if (skb->ip_summed == CHECKSUM_PARTIAL)
txd_info4 |= TX4_DMA_TUI_CO(7);
#endif
#if defined (CONFIG_RAETH_HW_VLAN_TX)
if (skb_vlan_tag_present(skb)) {
#if defined (RAETH_HW_VLAN4K)
txd_info4 |= (0x10000 | skb_vlan_tag_get(skb));
#else
u32 vlan_tci = skb_vlan_tag_get(skb);
txd_info4 |= (TX4_DMA_INSV | TX4_DMA_VPRI(vlan_tci));
txd_info4 |= (u32)ei_local->vlan_4k_map[(vlan_tci & VLAN_VID_MASK)];
#endif
}
#endif
#if defined (CONFIG_RAETH_SG_DMA_TX)
shinfo = skb_shinfo(skb);
#endif
#if defined (CONFIG_RAETH_TSO)
/* fill MSS info in tcp checksum field */
if (shinfo->gso_size) {
u32 hdr_len;
if (!(shinfo->gso_type & (SKB_GSO_TCPV4|SKB_GSO_TCPV6))) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
if (skb_header_cloned(skb)) {
if (pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
}
hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb));
if (hdr_len >= skb->len) {
dev_kfree_skb(skb);
return NETDEV_TX_OK;
}
tcp_hdr(skb)->check = htons(shinfo->gso_size);
txd_info4 |= TX4_DMA_TSO;
}
#endif
nr_desc = DIV_ROUND_UP(skb_headlen(skb), TXD_MAX_SEG_SIZE);
#if defined (CONFIG_RAETH_SG_DMA_TX)
nr_frags = (u32)shinfo->nr_frags;
for (i = 0; i < nr_frags; i++) {
tx_frag = &shinfo->frags[i];
nr_desc += DIV_ROUND_UP(skb_frag_size(tx_frag), TXD_MAX_SEG_SIZE);
}
#endif
nr_desc = DIV_ROUND_UP(nr_desc, 2);
txq = netdev_get_tx_queue(dev, 0);
/* flush main skb part before spin_lock() */
frag_size = (u32)skb_headlen(skb);
frag_addr = dma_map_single(NULL, skb->data, frag_size, DMA_TO_DEVICE);
/* protect TX ring access (from eth2/eth3 queues) */
spin_lock(&ei_local->page_lock);
/* check nr_desc+1 free descriptors */
next_idx = (ei_local->txd_last_idx + nr_desc) % NUM_TX_DESC;
if (ei_local->txd_buff[ei_local->txd_last_idx] || ei_local->txd_buff[next_idx]) {
spin_unlock(&ei_local->page_lock);
netif_tx_stop_queue(txq);
#if defined (CONFIG_RAETH_DEBUG)
if (net_ratelimit())
printk("%s: PDMA TX ring is full! (GMAC: %d)\n", RAETH_DEV_NAME, gmac_no);
#endif
return NETDEV_TX_BUSY;
}
pdma_write_skb_fragment(ei_local, frag_addr, frag_size, &desc_odd,
&txd_info2, txd_info4, skb, nr_frags == 0);
#if defined (CONFIG_RAETH_SG_DMA_TX)
for (i = 0; i < nr_frags; i++) {
tx_frag = &shinfo->frags[i];
frag_size = skb_frag_size(tx_frag);
frag_addr = skb_frag_dma_map(NULL, tx_frag, 0, frag_size, DMA_TO_DEVICE);
pdma_write_skb_fragment(ei_local, frag_addr, frag_size, &desc_odd,
&txd_info2, txd_info4, skb, i == nr_frags - 1);
}
#endif
#if defined (CONFIG_RAETH_BQL)
netdev_tx_sent_queue(txq, skb->len);
#endif
#if !defined (CONFIG_RAETH_BQL) || !defined (CONFIG_SMP)
/* smp_mb() already inlined in netdev_tx_sent_queue */
wmb();
#endif
/* kick the DMA TX */
sysRegWrite(TX_CTX_IDX0, cpu_to_le32(ei_local->txd_last_idx));
spin_unlock(&ei_local->page_lock);
return NETDEV_TX_OK;
}
inline void *chcr_crypto_wreq(struct sk_buff *skb,
struct net_device *dev,
void *pos,
int credits,
struct ipsec_sa_entry *sa_entry)
{
struct port_info *pi = netdev_priv(dev);
struct adapter *adap = pi->adapter;
unsigned int immdatalen = 0;
unsigned int ivsize = GCM_ESP_IV_SIZE;
struct chcr_ipsec_wr *wr;
unsigned int flits;
u32 wr_mid;
int qidx = skb_get_queue_mapping(skb);
struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];
unsigned int kctx_len = sa_entry->kctx_len;
int qid = q->q.cntxt_id;
atomic_inc(&adap->chcr_stats.ipsec_cnt);
flits = calc_tx_sec_flits(skb, kctx_len);
if (is_eth_imm(skb, kctx_len))
immdatalen = skb->len;
/* WR Header */
wr = (struct chcr_ipsec_wr *)pos;
wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
if (unlikely(credits < ETHTXQ_STOP_THRES)) {
netif_tx_stop_queue(q->txq);
q->q.stops++;
wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
}
wr_mid |= FW_ULPTX_WR_DATA_F;
wr->wreq.flowid_len16 = htonl(wr_mid);
/* ULPTX */
wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);
wr->req.ulptx.len = htonl(DIV_ROUND_UP(flits, 2) - 1);
/* Sub-command */
wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen);
wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
sizeof(wr->req.key_ctx) +
kctx_len +
sizeof(struct cpl_tx_pkt_core) +
immdatalen);
/* CPL_SEC_PDU */
wr->req.sec_cpl.op_ivinsrtofst = htonl(
CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
CPL_TX_SEC_PDU_CPLLEN_V(2) |
CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
CPL_TX_SEC_PDU_IVINSRTOFST_V(
(skb_transport_offset(skb) +
sizeof(struct ip_esp_hdr) + 1)));
wr->req.sec_cpl.pldlen = htonl(skb->len);
wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
(skb_transport_offset(skb) + 1),
(skb_transport_offset(skb) +
sizeof(struct ip_esp_hdr)),
(skb_transport_offset(skb) +
sizeof(struct ip_esp_hdr) +
GCM_ESP_IV_SIZE + 1), 0);
wr->req.sec_cpl.cipherstop_lo_authinsert =
FILL_SEC_CPL_AUTHINSERT(0, skb_transport_offset(skb) +
sizeof(struct ip_esp_hdr) +
GCM_ESP_IV_SIZE + 1,
sa_entry->authsize,
sa_entry->authsize);
wr->req.sec_cpl.seqno_numivs =
FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,
CHCR_SCMD_CIPHER_MODE_AES_GCM,
CHCR_SCMD_AUTH_MODE_GHASH,
sa_entry->hmac_ctrl,
ivsize >> 1);
wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
0, 0, 0);
pos += sizeof(struct fw_ulptx_wr) +
sizeof(struct ulp_txpkt) +
sizeof(struct ulptx_idata) +
sizeof(struct cpl_tx_sec_pdu);
pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);
return pos;
}
static inline int netvsc_send_pkt(
struct hv_netvsc_packet *packet,
struct netvsc_device *net_device)
{
struct nvsp_message nvmsg;
struct vmbus_channel *out_channel = packet->channel;
u16 q_idx = packet->q_idx;
struct net_device *ndev = net_device->ndev;
u64 req_id;
int ret;
struct hv_page_buffer *pgbuf;
u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound);
nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
if (packet->is_data_pkt) {
/* 0 is RMC_DATA; */
nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0;
} else {
/* 1 is RMC_CONTROL; */
nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1;
}
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index =
packet->send_buf_index;
if (packet->send_buf_index == NETVSC_INVALID_INDEX)
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0;
else
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size =
packet->total_data_buflen;
if (packet->send_completion)
req_id = (ulong)packet;
else
req_id = 0;
if (out_channel->rescind)
return -ENODEV;
/*
* It is possible that once we successfully place this packet
* on the ringbuffer, we may stop the queue. In that case, we want
* to notify the host independent of the xmit_more flag. We don't
* need to be precise here; in the worst case we may signal the host
* unnecessarily.
*/
if (ring_avail < (RING_AVAIL_PERCENT_LOWATER + 1))
packet->xmit_more = false;
if (packet->page_buf_cnt) {
pgbuf = packet->cp_partial ? packet->page_buf +
packet->rmsg_pgcnt : packet->page_buf;
ret = vmbus_sendpacket_pagebuffer_ctl(out_channel,
pgbuf,
packet->page_buf_cnt,
&nvmsg,
sizeof(struct nvsp_message),
req_id,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED,
!packet->xmit_more);
} else {
ret = vmbus_sendpacket_ctl(out_channel, &nvmsg,
sizeof(struct nvsp_message),
req_id,
VM_PKT_DATA_INBAND,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED,
!packet->xmit_more);
}
if (ret == 0) {
atomic_inc(&net_device->num_outstanding_sends);
atomic_inc(&net_device->queue_sends[q_idx]);
if (ring_avail < RING_AVAIL_PERCENT_LOWATER) {
netif_tx_stop_queue(netdev_get_tx_queue(ndev, q_idx));
if (atomic_read(&net_device->
queue_sends[q_idx]) < 1)
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
}
} else if (ret == -EAGAIN) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->queue_sends[q_idx]) < 1) {
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
ret = -ENOSPC;
}
} else {
netdev_err(ndev, "Unable to send packet %p ret %d\n",
packet, ret);
}
return ret;
}
static void eth_txq_stop(struct sge_eth_txq *q)
{
netif_tx_stop_queue(q->txq);
q->q.stops++;
}
/**
* efx_tx_queue_insert - push descriptors onto the TX queue
* @tx_queue: Efx TX queue
* @dma_addr: DMA address of fragment
* @len: Length of fragment
* @final_buffer: The final buffer inserted into the queue
*
* Push descriptors onto the TX queue. Return 0 on success or 1 if
* @tx_queue full.
*/
static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
dma_addr_t dma_addr, unsigned len,
struct efx_tx_buffer **final_buffer)
{
struct efx_tx_buffer *buffer;
struct efx_nic *efx = tx_queue->efx;
unsigned dma_len, fill_level, insert_ptr;
int q_space;
EFX_BUG_ON_PARANOID(len <= 0);
fill_level = tx_queue->insert_count - tx_queue->old_read_count;
/* -1 as there is no way to represent all descriptors used */
q_space = efx->txq_entries - 1 - fill_level;
while (1) {
if (unlikely(q_space-- <= 0)) {
/* It might be that completions have happened
* since the xmit path last checked. Update
* the xmit path's copy of read_count.
*/
netif_tx_stop_queue(tx_queue->core_txq);
/* This memory barrier protects the change of
* queue state from the access of read_count. */
smp_mb();
tx_queue->old_read_count =
ACCESS_ONCE(tx_queue->read_count);
fill_level = (tx_queue->insert_count
- tx_queue->old_read_count);
q_space = efx->txq_entries - 1 - fill_level;
if (unlikely(q_space-- <= 0)) {
*final_buffer = NULL;
return 1;
}
smp_mb();
netif_tx_start_queue(tx_queue->core_txq);
}
insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
buffer = &tx_queue->buffer[insert_ptr];
++tx_queue->insert_count;
EFX_BUG_ON_PARANOID(tx_queue->insert_count -
tx_queue->read_count >=
efx->txq_entries);
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(!buffer->continuation);
EFX_BUG_ON_PARANOID(buffer->tsoh);
buffer->dma_addr = dma_addr;
dma_len = efx_max_tx_len(efx, dma_addr);
/* If there is enough space to send then do so */
if (dma_len >= len)
break;
buffer->len = dma_len; /* Don't set the other members */
dma_addr += dma_len;
len -= dma_len;
}
EFX_BUG_ON_PARANOID(!len);
buffer->len = len;
*final_buffer = buffer;
return 0;
}
/*
* Add a socket buffer to a TX queue
*
* This maps all fragments of a socket buffer for DMA and adds them to
* the TX queue. The queue's insert pointer will be incremented by
* the number of fragments in the socket buffer.
*
* If any DMA mapping fails, any mapped fragments will be unmapped,
* the queue's insert pointer will be restored to its original value.
*
* This function is split out from efx_hard_start_xmit to allow the
* loopback test to direct packets via specific TX queues.
*
* Returns NETDEV_TX_OK or NETDEV_TX_BUSY
* You must hold netif_tx_lock() to call this function.
*/
netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
{
struct efx_nic *efx = tx_queue->efx;
struct pci_dev *pci_dev = efx->pci_dev;
struct efx_tx_buffer *buffer;
skb_frag_t *fragment;
struct page *page;
int page_offset;
unsigned int len, unmap_len = 0, fill_level, insert_ptr;
dma_addr_t dma_addr, unmap_addr = 0;
unsigned int dma_len;
bool unmap_single;
int q_space, i = 0;
netdev_tx_t rc = NETDEV_TX_OK;
EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
if (skb_shinfo(skb)->gso_size)
return efx_enqueue_skb_tso(tx_queue, skb);
/* Get size of the initial fragment */
len = skb_headlen(skb);
/* Pad if necessary */
if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) {
EFX_BUG_ON_PARANOID(skb->data_len);
len = 32 + 1;
if (skb_pad(skb, len - skb->len))
return NETDEV_TX_OK;
}
fill_level = tx_queue->insert_count - tx_queue->old_read_count;
q_space = efx->txq_entries - 1 - fill_level;
/* Map for DMA. Use pci_map_single rather than pci_map_page
* since this is more efficient on machines with sparse
* memory.
*/
unmap_single = true;
dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
/* Process all fragments */
while (1) {
if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
goto pci_err;
/* Store fields for marking in the per-fragment final
* descriptor */
unmap_len = len;
unmap_addr = dma_addr;
/* Add to TX queue, splitting across DMA boundaries */
do {
if (unlikely(q_space-- <= 0)) {
/* It might be that completions have
* happened since the xmit path last
* checked. Update the xmit path's
* copy of read_count.
*/
netif_tx_stop_queue(tx_queue->core_txq);
/* This memory barrier protects the
* change of queue state from the access
* of read_count. */
smp_mb();
tx_queue->old_read_count =
ACCESS_ONCE(tx_queue->read_count);
fill_level = (tx_queue->insert_count
- tx_queue->old_read_count);
q_space = efx->txq_entries - 1 - fill_level;
if (unlikely(q_space-- <= 0)) {
rc = NETDEV_TX_BUSY;
goto unwind;
}
smp_mb();
netif_tx_start_queue(tx_queue->core_txq);
}
insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
buffer = &tx_queue->buffer[insert_ptr];
efx_tsoh_free(tx_queue, buffer);
EFX_BUG_ON_PARANOID(buffer->tsoh);
EFX_BUG_ON_PARANOID(buffer->skb);
EFX_BUG_ON_PARANOID(buffer->len);
EFX_BUG_ON_PARANOID(!buffer->continuation);
EFX_BUG_ON_PARANOID(buffer->unmap_len);
dma_len = efx_max_tx_len(efx, dma_addr);
if (likely(dma_len >= len))
dma_len = len;
/* Fill out per descriptor fields */
buffer->len = dma_len;
buffer->dma_addr = dma_addr;
len -= dma_len;
dma_addr += dma_len;
++tx_queue->insert_count;
} while (len);
/* Transfer ownership of the unmapping to the final buffer */
buffer->unmap_single = unmap_single;
buffer->unmap_len = unmap_len;
unmap_len = 0;
/* Get address and size of next fragment */
if (i >= skb_shinfo(skb)->nr_frags)
break;
fragment = &skb_shinfo(skb)->frags[i];
len = fragment->size;
page = fragment->page;
page_offset = fragment->page_offset;
i++;
/* Map for DMA */
unmap_single = false;
dma_addr = pci_map_page(pci_dev, page, page_offset, len,
PCI_DMA_TODEVICE);
}
/* Transfer ownership of the skb to the final buffer */
buffer->skb = skb;
buffer->continuation = false;
/* Pass off to hardware */
efx_nic_push_buffers(tx_queue);
return NETDEV_TX_OK;
pci_err:
netif_err(efx, tx_err, efx->net_dev,
" TX queue %d could not map skb with %d bytes %d "
"fragments for DMA\n", tx_queue->queue, skb->len,
skb_shinfo(skb)->nr_frags + 1);
/* Mark the packet as transmitted, and free the SKB ourselves */
dev_kfree_skb_any(skb);
unwind:
/* Work backwards until we hit the original insert pointer value */
while (tx_queue->insert_count != tx_queue->write_count) {
--tx_queue->insert_count;
insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
buffer = &tx_queue->buffer[insert_ptr];
efx_dequeue_buffer(tx_queue, buffer);
buffer->len = 0;
}
/* Free the fragment we were mid-way through pushing */
if (unmap_len) {
if (unmap_single)
pci_unmap_single(pci_dev, unmap_addr, unmap_len,
PCI_DMA_TODEVICE);
else
pci_unmap_page(pci_dev, unmap_addr, unmap_len,
PCI_DMA_TODEVICE);
}
return rc;
}
static inline void netif_tx_disable_queue(struct netdev_queue *txq)
{
__netif_tx_lock_bh(txq);
netif_tx_stop_queue(txq);
__netif_tx_unlock_bh(txq);
}
static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
{
struct mlx5_wq_cyc *wq = &sq->wq;
u16 pi = sq->pc & wq->sz_m1;
struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
struct mlx5_wqe_data_seg *dseg;
u8 opcode = MLX5_OPCODE_SEND;
dma_addr_t dma_addr = 0;
bool bf = false;
u16 headlen;
u16 ds_cnt;
u16 ihs;
int i;
memset(wqe, 0, sizeof(*wqe));
if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM;
else
sq->stats.csum_offload_none++;
if (sq->cc != sq->prev_cc) {
sq->prev_cc = sq->cc;
sq->bf_budget = (sq->cc == sq->pc) ? MLX5E_SQ_BF_BUDGET : 0;
}
if (skb_is_gso(skb)) {
u32 payload_len;
eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
opcode = MLX5_OPCODE_LSO;
ihs = skb_transport_offset(skb) + tcp_hdrlen(skb);
payload_len = skb->len - ihs;
MLX5E_TX_SKB_CB(skb)->num_bytes = skb->len +
(skb_shinfo(skb)->gso_segs - 1) * ihs;
sq->stats.tso_packets++;
sq->stats.tso_bytes += payload_len;
} else {
bf = sq->bf_budget &&
!skb->xmit_more &&
!skb_shinfo(skb)->nr_frags;
ihs = mlx5e_get_inline_hdr_size(sq, skb, bf);
MLX5E_TX_SKB_CB(skb)->num_bytes = max_t(unsigned int, skb->len,
ETH_ZLEN);
}
skb_copy_from_linear_data(skb, eseg->inline_hdr_start, ihs);
skb_pull_inline(skb, ihs);
eseg->inline_hdr_sz = cpu_to_be16(ihs);
ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start),
MLX5_SEND_WQE_DS);
dseg = (struct mlx5_wqe_data_seg *)cseg + ds_cnt;
MLX5E_TX_SKB_CB(skb)->num_dma = 0;
headlen = skb_headlen(skb);
if (headlen) {
dma_addr = dma_map_single(sq->pdev, skb->data, headlen,
DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
goto dma_unmap_wqe_err;
dseg->addr = cpu_to_be64(dma_addr);
dseg->lkey = sq->mkey_be;
dseg->byte_count = cpu_to_be32(headlen);
mlx5e_dma_push(sq, dma_addr, headlen);
MLX5E_TX_SKB_CB(skb)->num_dma++;
dseg++;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
int fsz = skb_frag_size(frag);
dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
goto dma_unmap_wqe_err;
dseg->addr = cpu_to_be64(dma_addr);
dseg->lkey = sq->mkey_be;
dseg->byte_count = cpu_to_be32(fsz);
mlx5e_dma_push(sq, dma_addr, fsz);
MLX5E_TX_SKB_CB(skb)->num_dma++;
dseg++;
}
ds_cnt += MLX5E_TX_SKB_CB(skb)->num_dma;
cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
sq->skb[pi] = skb;
MLX5E_TX_SKB_CB(skb)->num_wqebbs = DIV_ROUND_UP(ds_cnt,
MLX5_SEND_WQEBB_NUM_DS);
sq->pc += MLX5E_TX_SKB_CB(skb)->num_wqebbs;
netdev_tx_sent_queue(sq->txq, MLX5E_TX_SKB_CB(skb)->num_bytes);
if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM))) {
netif_tx_stop_queue(sq->txq);
sq->stats.stopped++;
}
if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) {
int bf_sz = 0;
if (bf && sq->uar_bf_map)
bf_sz = MLX5E_TX_SKB_CB(skb)->num_wqebbs << 3;
cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
mlx5e_tx_notify_hw(sq, wqe, bf_sz);
}
/* fill sq edge with nops to avoid wqe wrap around */
while ((sq->pc & wq->sz_m1) > sq->edge)
mlx5e_send_nop(sq, false);
sq->bf_budget = bf ? sq->bf_budget - 1 : 0;
sq->stats.packets++;
return NETDEV_TX_OK;
dma_unmap_wqe_err:
sq->stats.dropped++;
mlx5e_dma_unmap_wqe_err(sq, skb);
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb)
{
struct mlx5_wq_cyc *wq = &sq->wq;
u16 pi = sq->pc & wq->sz_m1;
struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
struct mlx5e_tx_wqe_info *wi = &sq->wqe_info[pi];
struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
struct mlx5_wqe_data_seg *dseg;
unsigned char *skb_data = skb->data;
unsigned int skb_len = skb->len;
u8 opcode = MLX5_OPCODE_SEND;
dma_addr_t dma_addr = 0;
unsigned int num_bytes;
bool bf = false;
u16 headlen;
u16 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
u16 ihs;
int i;
memset(wqe, 0, sizeof(*wqe));
if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM;
if (skb->encapsulation) {
eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM |
MLX5_ETH_WQE_L4_INNER_CSUM;
sq->stats.csum_partial_inner++;
} else {
eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM;
sq->stats.csum_partial++;
}
} else
sq->stats.csum_none++;
if (sq->cc != sq->prev_cc) {
sq->prev_cc = sq->cc;
sq->bf_budget = (sq->cc == sq->pc) ? MLX5E_SQ_BF_BUDGET : 0;
}
if (skb_is_gso(skb)) {
eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
opcode = MLX5_OPCODE_LSO;
if (skb->encapsulation) {
ihs = skb_inner_transport_header(skb) - skb->data +
inner_tcp_hdrlen(skb);
sq->stats.tso_inner_packets++;
sq->stats.tso_inner_bytes += skb->len - ihs;
} else {
ihs = skb_transport_offset(skb) + tcp_hdrlen(skb);
sq->stats.tso_packets++;
sq->stats.tso_bytes += skb->len - ihs;
}
num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs;
} else {
bf = sq->bf_budget &&
!skb->xmit_more &&
!skb_shinfo(skb)->nr_frags;
ihs = mlx5e_get_inline_hdr_size(sq, skb, bf);
num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
}
wi->num_bytes = num_bytes;
if (skb_vlan_tag_present(skb)) {
mlx5e_insert_vlan(eseg->inline_hdr_start, skb, ihs, &skb_data,
&skb_len);
ihs += VLAN_HLEN;
} else {
memcpy(eseg->inline_hdr_start, skb_data, ihs);
mlx5e_tx_skb_pull_inline(&skb_data, &skb_len, ihs);
}
eseg->inline_hdr_sz = cpu_to_be16(ihs);
ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start),
MLX5_SEND_WQE_DS);
dseg = (struct mlx5_wqe_data_seg *)cseg + ds_cnt;
wi->num_dma = 0;
headlen = skb_len - skb->data_len;
if (headlen) {
dma_addr = dma_map_single(sq->pdev, skb_data, headlen,
DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
goto dma_unmap_wqe_err;
dseg->addr = cpu_to_be64(dma_addr);
dseg->lkey = sq->mkey_be;
dseg->byte_count = cpu_to_be32(headlen);
mlx5e_dma_push(sq, dma_addr, headlen, MLX5E_DMA_MAP_SINGLE);
wi->num_dma++;
dseg++;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
int fsz = skb_frag_size(frag);
dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
goto dma_unmap_wqe_err;
dseg->addr = cpu_to_be64(dma_addr);
dseg->lkey = sq->mkey_be;
dseg->byte_count = cpu_to_be32(fsz);
mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE);
wi->num_dma++;
dseg++;
}
ds_cnt += wi->num_dma;
cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
sq->skb[pi] = skb;
wi->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
sq->pc += wi->num_wqebbs;
if (unlikely(MLX5E_TX_HW_STAMP(sq->channel->priv, skb)))
skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
netdev_tx_sent_queue(sq->txq, wi->num_bytes);
if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM))) {
netif_tx_stop_queue(sq->txq);
sq->stats.queue_stopped++;
}
if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) {
int bf_sz = 0;
if (bf && sq->uar_bf_map)
bf_sz = wi->num_wqebbs << 3;
cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
mlx5e_tx_notify_hw(sq, &wqe->ctrl, bf_sz);
}
sq->bf_budget = bf ? sq->bf_budget - 1 : 0;
/* fill sq edge with nops to avoid wqe wrap around */
while ((sq->pc & wq->sz_m1) > sq->edge)
mlx5e_send_nop(sq, false);
sq->stats.packets++;
sq->stats.bytes += num_bytes;
return NETDEV_TX_OK;
dma_unmap_wqe_err:
sq->stats.queue_dropped++;
mlx5e_dma_unmap_wqe_err(sq, wi->num_dma);
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
/**============================================================================
@brief hdd_hard_start_xmit() - Function registered with the Linux OS for
transmitting packets. There are 2 versions of this function. One that uses
locked queue and other that uses lockless queues. Both have been retained to
do some performance testing
@param skb : [in] pointer to OS packet (sk_buff)
@param dev : [in] pointer to Libra network device
@return : NET_XMIT_DROP if packets are dropped
: NET_XMIT_SUCCESS if packet is enqueued succesfully
===========================================================================*/
int hdd_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
VOS_STATUS status;
WLANTL_ACEnumType ac;
sme_QosWmmUpType up;
skb_list_node_t *pktNode = NULL;
hdd_list_node_t *anchor = NULL;
v_SIZE_t pktListSize = 0;
hdd_adapter_t *pAdapter = WLAN_HDD_GET_PRIV_PTR(dev);
v_BOOL_t granted;
hdd_station_ctx_t *pHddStaCtx = &pAdapter->sessionCtx.station;
v_BOOL_t txSuspended = VOS_FALSE;
++pAdapter->hdd_stats.hddTxRxStats.txXmitCalled;
//Get TL AC corresponding to Qdisc queue index/AC.
ac = hdd_QdiscAcToTlAC[skb->queue_mapping];
//user priority from IP header, which is already extracted and set from
//select_queue call back function
up = skb->priority;
++pAdapter->hdd_stats.hddTxRxStats.txXmitClassifiedAC[ac];
#ifdef HDD_WMM_DEBUG
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL,
"%s: Classified as ac %d up %d", __FUNCTION__, ac, up);
#endif // HDD_WMM_DEBUG
spin_lock(&pAdapter->wmm_tx_queue[ac].lock);
/*For every increment of 10 pkts in the queue, we inform TL about pending pkts.
* We check for +1 in the logic,to take care of Zero count which
* occurs very frequently in low traffic cases */
if((pAdapter->wmm_tx_queue[ac].count + 1) % 10 == 0)
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_INFO,"%s:Queue is Filling up.Inform TL again about pending packets", __FUNCTION__);
WLANTL_STAPktPending( (WLAN_HDD_GET_CTX(pAdapter))->pvosContext, pHddStaCtx->conn_info.staId[0], ac );
}
//If we have already reached the max queue size, disable the TX queue
if ( pAdapter->wmm_tx_queue[ac].count == pAdapter->wmm_tx_queue[ac].max_size)
{
++pAdapter->hdd_stats.hddTxRxStats.txXmitBackPressured;
++pAdapter->hdd_stats.hddTxRxStats.txXmitBackPressuredAC[ac];
netif_tx_stop_queue(netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)));
pAdapter->isTxSuspended[ac] = VOS_TRUE;
txSuspended = VOS_TRUE;
}
spin_unlock(&pAdapter->wmm_tx_queue[ac].lock);
if (VOS_TRUE == txSuspended)
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,
"%s: TX queue full for AC=%d Disable OS TX queue",
__FUNCTION__, ac );
return NETDEV_TX_BUSY;
}
//Use the skb->cb field to hold the list node information
pktNode = (skb_list_node_t *)&skb->cb;
//Stick the OS packet inside this node.
pktNode->skb = skb;
//Stick the User Priority inside this node
pktNode->userPriority = up;
INIT_LIST_HEAD(&pktNode->anchor);
//Insert the OS packet into the appropriate AC queue
spin_lock(&pAdapter->wmm_tx_queue[ac].lock);
status = hdd_list_insert_back_size( &pAdapter->wmm_tx_queue[ac], &pktNode->anchor, &pktListSize );
spin_unlock(&pAdapter->wmm_tx_queue[ac].lock);
if ( !VOS_IS_STATUS_SUCCESS( status ) )
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,"%s:Insert Tx queue failed. Pkt dropped", __FUNCTION__);
++pAdapter->hdd_stats.hddTxRxStats.txXmitDropped;
++pAdapter->hdd_stats.hddTxRxStats.txXmitDroppedAC[ac];
++pAdapter->stats.tx_dropped;
kfree_skb(skb);
return NETDEV_TX_OK;
}
++pAdapter->hdd_stats.hddTxRxStats.txXmitQueued;
++pAdapter->hdd_stats.hddTxRxStats.txXmitQueuedAC[ac];
//Make sure we have access to this access category
if (likely(pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcAccessAllowed) ||
( pHddStaCtx->conn_info.uIsAuthenticated == VOS_FALSE))
{
granted = VOS_TRUE;
}
else
{
status = hdd_wmm_acquire_access( pAdapter, ac, &granted );
}
if ( granted && ( pktListSize == 1 ))
{
//Let TL know we have a packet to send for this AC
//VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR,"%s:Indicating Packet to TL", __FUNCTION__);
status = WLANTL_STAPktPending( (WLAN_HDD_GET_CTX(pAdapter))->pvosContext, pHddStaCtx->conn_info.staId[0], ac );
if ( !VOS_IS_STATUS_SUCCESS( status ) )
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN, "%s: Failed to signal TL for AC=%d", __FUNCTION__, ac );
//Remove the packet from queue. It must be at the back of the queue, as TX thread cannot preempt us in the middle
//as we are in a soft irq context. Also it must be the same packet that we just allocated.
spin_lock(&pAdapter->wmm_tx_queue[ac].lock);
status = hdd_list_remove_back( &pAdapter->wmm_tx_queue[ac], &anchor );
spin_unlock(&pAdapter->wmm_tx_queue[ac].lock);
++pAdapter->stats.tx_dropped;
++pAdapter->hdd_stats.hddTxRxStats.txXmitDropped;
++pAdapter->hdd_stats.hddTxRxStats.txXmitDroppedAC[ac];
kfree_skb(skb);
return NETDEV_TX_OK;
}
}
dev->trans_start = jiffies;
return NETDEV_TX_OK;
}
static inline int netvsc_send_pkt(
struct hv_netvsc_packet *packet,
struct netvsc_device *net_device)
{
struct nvsp_message nvmsg;
struct vmbus_channel *out_channel = packet->channel;
u16 q_idx = packet->q_idx;
struct net_device *ndev = net_device->ndev;
u64 req_id;
int ret;
struct hv_page_buffer *pgbuf;
nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
if (packet->is_data_pkt) {
/* 0 is RMC_DATA; */
nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0;
} else {
/* 1 is RMC_CONTROL; */
nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1;
}
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index =
packet->send_buf_index;
if (packet->send_buf_index == NETVSC_INVALID_INDEX)
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0;
else
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size =
packet->total_data_buflen;
if (packet->send_completion)
req_id = (ulong)packet;
else
req_id = 0;
if (out_channel->rescind)
return -ENODEV;
if (packet->page_buf_cnt) {
pgbuf = packet->cp_partial ? packet->page_buf +
packet->rmsg_pgcnt : packet->page_buf;
ret = vmbus_sendpacket_pagebuffer(out_channel,
pgbuf,
packet->page_buf_cnt,
&nvmsg,
sizeof(struct nvsp_message),
req_id);
} else {
ret = vmbus_sendpacket(
out_channel, &nvmsg,
sizeof(struct nvsp_message),
req_id,
VM_PKT_DATA_INBAND,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
}
if (ret == 0) {
atomic_inc(&net_device->num_outstanding_sends);
atomic_inc(&net_device->queue_sends[q_idx]);
if (hv_ringbuf_avail_percent(&out_channel->outbound) <
RING_AVAIL_PERCENT_LOWATER) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->
queue_sends[q_idx]) < 1)
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
}
} else if (ret == -EAGAIN) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->queue_sends[q_idx]) < 1) {
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
ret = -ENOSPC;
}
} else {
netdev_err(ndev, "Unable to send packet %p ret %d\n",
packet, ret);
}
return ret;
}