void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index) { unsigned fill_level; struct efx_nic *efx = tx_queue->efx; EFX_BUG_ON_PARANOID(index > tx_queue->ptr_mask); efx_dequeue_buffers(tx_queue, index); /* See if we need to restart the netif queue. This barrier * separates the update of read_count from the test of the * queue state. */ smp_mb(); if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) && likely(efx->port_enabled)) { fill_level = tx_queue->insert_count - tx_queue->read_count; if (fill_level < EFX_TXQ_THRESHOLD(efx)) { EFX_BUG_ON_PARANOID(!efx_dev_registered(efx)); netif_tx_wake_queue(tx_queue->core_txq); } } /* Check whether the hardware queue is now empty */ if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) { tx_queue->old_write_count = ACCESS_ONCE(tx_queue->write_count); if (tx_queue->read_count == tx_queue->old_write_count) { smp_mb(); tx_queue->empty_read_count = tx_queue->read_count | EFX_EMPTY_COUNT_VALID; } } }
/* * Packet send completion callback handler. * * It either frees the buffer directly or forwards it to another * completion callback which checks conditions, updates statistics, * wakes up stalled traffic queue if required, and then frees the buffer. */ int mwifiex_write_data_complete(struct mwifiex_adapter *adapter, struct sk_buff *skb, int aggr, int status) { struct mwifiex_private *priv; struct mwifiex_txinfo *tx_info; struct netdev_queue *txq; int index; if (!skb) return 0; tx_info = MWIFIEX_SKB_TXCB(skb); priv = mwifiex_get_priv_by_id(adapter, tx_info->bss_num, tx_info->bss_type); if (!priv) goto done; if (adapter->iface_type == MWIFIEX_USB) adapter->data_sent = false; mwifiex_set_trans_start(priv->netdev); if (!status) { priv->stats.tx_packets++; priv->stats.tx_bytes += skb->len; if (priv->tx_timeout_cnt) priv->tx_timeout_cnt = 0; } else { priv->stats.tx_errors++; } if (tx_info->flags & MWIFIEX_BUF_FLAG_BRIDGED_PKT) atomic_dec_return(&adapter->pending_bridged_pkts); if (aggr) /* For skb_aggr, do not wake up tx queue */ goto done; atomic_dec(&adapter->tx_pending); index = mwifiex_1d_to_wmm_queue[skb->priority]; if (atomic_dec_return(&priv->wmm_tx_pending[index]) < LOW_TX_PENDING) { txq = netdev_get_tx_queue(priv->netdev, index); if (netif_tx_queue_stopped(txq)) { netif_tx_wake_queue(txq); dev_dbg(adapter->dev, "wake queue: %d\n", index); } } done: dev_kfree_skb_any(skb); return 0; }
/* * This function wakes up all queues in net_device */ void mwifiex_wake_up_net_dev_queue(struct net_device *netdev, struct mwifiex_adapter *adapter) { unsigned long dev_queue_flags; unsigned int i; spin_lock_irqsave(&adapter->queue_lock, dev_queue_flags); for (i = 0; i < netdev->num_tx_queues; i++) { struct netdev_queue *txq = netdev_get_tx_queue(netdev, i); if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); } spin_unlock_irqrestore(&adapter->queue_lock, dev_queue_flags); }
static int xlgmac_tx_poll(struct xlgmac_channel *channel) { struct xlgmac_pdata *pdata = channel->pdata; struct xlgmac_ring *ring = channel->tx_ring; struct net_device *netdev = pdata->netdev; unsigned int tx_packets = 0, tx_bytes = 0; struct xlgmac_desc_data *desc_data; struct xlgmac_dma_desc *dma_desc; struct xlgmac_desc_ops *desc_ops; struct xlgmac_hw_ops *hw_ops; struct netdev_queue *txq; int processed = 0; unsigned int cur; desc_ops = &pdata->desc_ops; hw_ops = &pdata->hw_ops; /* Nothing to do if there isn't a Tx ring for this channel */ if (!ring) return 0; cur = ring->cur; /* Be sure we get ring->cur before accessing descriptor data */ smp_rmb(); txq = netdev_get_tx_queue(netdev, channel->queue_index); while ((processed < XLGMAC_TX_DESC_MAX_PROC) && (ring->dirty != cur)) { desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty); dma_desc = desc_data->dma_desc; if (!hw_ops->tx_complete(dma_desc)) break; /* Make sure descriptor fields are read after reading * the OWN bit */ dma_rmb(); if (netif_msg_tx_done(pdata)) xlgmac_dump_tx_desc(pdata, ring, ring->dirty, 1, 0); if (hw_ops->is_last_desc(dma_desc)) { tx_packets += desc_data->tx.packets; tx_bytes += desc_data->tx.bytes; } /* Free the SKB and reset the descriptor for re-use */ desc_ops->unmap_desc_data(pdata, desc_data); hw_ops->tx_desc_reset(desc_data); processed++; ring->dirty++; } if (!processed) return 0; netdev_tx_completed_queue(txq, tx_packets, tx_bytes); if ((ring->tx.queue_stopped == 1) && (xlgmac_tx_avail_desc(ring) > XLGMAC_TX_DESC_MIN_FREE)) { ring->tx.queue_stopped = 0; netif_tx_wake_queue(txq); } XLGMAC_PR("processed=%d\n", processed); return processed; }
static bool mlx4_en_process_tx_cq(struct ether *dev, struct mlx4_en_cq *cq) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cq *mcq = &cq->mcq; struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring]; struct mlx4_cqe *cqe; uint16_t index; uint16_t new_index, ring_index, stamp_index; uint32_t txbbs_skipped = 0; uint32_t txbbs_stamp = 0; uint32_t cons_index = mcq->cons_index; int size = cq->size; uint32_t size_mask = ring->size_mask; struct mlx4_cqe *buf = cq->buf; uint32_t packets = 0; uint32_t bytes = 0; int factor = priv->cqe_factor; uint64_t timestamp = 0; int done = 0; int budget = priv->tx_work_limit; uint32_t last_nr_txbb; uint32_t ring_cons; if (!priv->port_up) return true; #if 0 // AKAROS_PORT netdev_txq_bql_complete_prefetchw(ring->tx_queue); #endif index = cons_index & size_mask; cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb); ring_cons = ACCESS_ONCE(ring->cons); ring_index = ring_cons & size_mask; stamp_index = ring_index; /* Process all completed CQEs */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size) && (done < budget)) { /* * make sure we read the CQE after we read the * ownership bit */ bus_rmb(); if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_ERROR)) { struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe; en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n", cqe_err->vendor_err_syndrome, cqe_err->syndrome); } /* Skip over last polled CQE */ new_index = be16_to_cpu(cqe->wqe_index) & size_mask; do { txbbs_skipped += last_nr_txbb; ring_index = (ring_index + last_nr_txbb) & size_mask; if (ring->tx_info[ring_index].ts_requested) timestamp = mlx4_en_get_cqe_ts(cqe); /* free next descriptor */ last_nr_txbb = mlx4_en_free_tx_desc( priv, ring, ring_index, !!((ring_cons + txbbs_skipped) & ring->size), timestamp); mlx4_en_stamp_wqe(priv, ring, stamp_index, !!((ring_cons + txbbs_stamp) & ring->size)); stamp_index = ring_index; txbbs_stamp = txbbs_skipped; packets++; bytes += ring->tx_info[ring_index].nr_bytes; } while ((++done < budget) && (ring_index != new_index)); ++cons_index; index = cons_index & size_mask; cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; } /* * To prevent CQ overflow we first update CQ consumer and only then * the ring consumer. */ mcq->cons_index = cons_index; mlx4_cq_set_ci(mcq); wmb(); /* we want to dirty this cache line once */ ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb; ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped; #if 0 // AKAROS_PORT netdev_tx_completed_queue(ring->tx_queue, packets, bytes); /* * Wakeup Tx queue if this stopped, and at least 1 packet * was completed */ if (netif_tx_queue_stopped(ring->tx_queue) && txbbs_skipped > 0) { netif_tx_wake_queue(ring->tx_queue); ring->wake_queue++; } #endif return done < budget; }
static inline int netvsc_send_pkt( struct hv_netvsc_packet *packet, struct netvsc_device *net_device) { struct nvsp_message nvmsg; struct vmbus_channel *out_channel = packet->channel; u16 q_idx = packet->q_idx; struct net_device *ndev = net_device->ndev; u64 req_id; int ret; struct hv_page_buffer *pgbuf; u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound); nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; if (packet->is_data_pkt) { /* 0 is RMC_DATA; */ nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0; } else { /* 1 is RMC_CONTROL; */ nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1; } nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index = packet->send_buf_index; if (packet->send_buf_index == NETVSC_INVALID_INDEX) nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0; else nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = packet->total_data_buflen; if (packet->send_completion) req_id = (ulong)packet; else req_id = 0; if (out_channel->rescind) return -ENODEV; /* * It is possible that once we successfully place this packet * on the ringbuffer, we may stop the queue. In that case, we want * to notify the host independent of the xmit_more flag. We don't * need to be precise here; in the worst case we may signal the host * unnecessarily. */ if (ring_avail < (RING_AVAIL_PERCENT_LOWATER + 1)) packet->xmit_more = false; if (packet->page_buf_cnt) { pgbuf = packet->cp_partial ? packet->page_buf + packet->rmsg_pgcnt : packet->page_buf; ret = vmbus_sendpacket_pagebuffer_ctl(out_channel, pgbuf, packet->page_buf_cnt, &nvmsg, sizeof(struct nvsp_message), req_id, VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED, !packet->xmit_more); } else { ret = vmbus_sendpacket_ctl(out_channel, &nvmsg, sizeof(struct nvsp_message), req_id, VM_PKT_DATA_INBAND, VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED, !packet->xmit_more); } if (ret == 0) { atomic_inc(&net_device->num_outstanding_sends); atomic_inc(&net_device->queue_sends[q_idx]); if (ring_avail < RING_AVAIL_PERCENT_LOWATER) { netif_tx_stop_queue(netdev_get_tx_queue(ndev, q_idx)); if (atomic_read(&net_device-> queue_sends[q_idx]) < 1) netif_tx_wake_queue(netdev_get_tx_queue( ndev, q_idx)); } } else if (ret == -EAGAIN) { netif_tx_stop_queue(netdev_get_tx_queue( ndev, q_idx)); if (atomic_read(&net_device->queue_sends[q_idx]) < 1) { netif_tx_wake_queue(netdev_get_tx_queue( ndev, q_idx)); ret = -ENOSPC; } } else { netdev_err(ndev, "Unable to send packet %p ret %d\n", packet, ret); } return ret; }
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq) { struct mlx5e_sq *sq; u32 dma_fifo_cc; u32 nbytes; u16 npkts; u16 sqcc; int i; /* avoid accessing cq (dma coherent memory) if not needed */ if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags)) return false; sq = container_of(cq, struct mlx5e_sq, cq); npkts = 0; nbytes = 0; /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), * otherwise a cq overrun may occur */ sqcc = sq->cc; /* avoid dirtying sq cache line every cqe */ dma_fifo_cc = sq->dma_fifo_cc; for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { struct mlx5_cqe64 *cqe; u16 wqe_counter; bool last_wqe; cqe = mlx5e_get_cqe(cq); if (!cqe) break; mlx5_cqwq_pop(&cq->wq); wqe_counter = be16_to_cpu(cqe->wqe_counter); do { struct sk_buff *skb; u16 ci; int j; last_wqe = (sqcc == wqe_counter); ci = sqcc & sq->wq.sz_m1; skb = sq->skb[ci]; if (unlikely(!skb)) { /* nop */ sq->stats.nop++; sqcc++; continue; } for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) { dma_addr_t addr; u32 size; mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size); dma_fifo_cc++; dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE); } npkts++; nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes; sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs; dev_kfree_skb(skb); } while (!last_wqe); } mlx5_cqwq_update_db_record(&cq->wq); /* ensure cq space is freed before enabling more cqes */ wmb(); sq->dma_fifo_cc = dma_fifo_cc; sq->cc = sqcc; netdev_tx_completed_queue(sq->txq, npkts, nbytes); if (netif_tx_queue_stopped(sq->txq) && mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) && likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) { netif_tx_wake_queue(sq->txq); sq->stats.wake++; } if (i == MLX5E_TX_CQ_POLL_BUDGET) { set_bit(MLX5E_CQ_HAS_CQES, &cq->flags); return true; } return false; }
static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cq *mcq = &cq->mcq; struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; struct mlx4_cqe *cqe = cq->buf; u16 index; u16 new_index; u32 txbbs_skipped = 0; u32 cq_last_sav; /* index always points to the first TXBB of the last polled descriptor */ index = ring->cons & ring->size_mask; new_index = be16_to_cpu(cqe->wqe_index) & ring->size_mask; if (index == new_index) return; if (!priv->port_up) return; /* * We use a two-stage loop: * - the first samples the HW-updated CQE * - the second frees TXBBs until the last sample * This lets us amortize CQE cache misses, while still polling the CQ * until is quiescent. */ cq_last_sav = mcq->cons_index; do { do { /* Skip over last polled CQE */ index = (index + ring->last_nr_txbb) & ring->size_mask; txbbs_skipped += ring->last_nr_txbb; /* Poll next CQE */ ring->last_nr_txbb = mlx4_en_free_tx_desc( priv, ring, index, !!((ring->cons + txbbs_skipped) & ring->size)); ++mcq->cons_index; } while (index != new_index); new_index = be16_to_cpu(cqe->wqe_index) & ring->size_mask; } while (index != new_index); AVG_PERF_COUNTER(priv->pstats.tx_coal_avg, (u32) (mcq->cons_index - cq_last_sav)); /* * To prevent CQ overflow we first update CQ consumer and only then * the ring consumer. */ mlx4_cq_set_ci(mcq); wmb(); ring->cons += txbbs_skipped; /* Wakeup Tx queue if this ring stopped it */ if (unlikely(ring->blocked)) { if ((u32) (ring->prod - ring->cons) <= ring->size - HEADROOM - MAX_DESC_TXBBS) { ring->blocked = 0; netif_tx_wake_queue(netdev_get_tx_queue(dev, cq->ring)); priv->port_stats.wake_queue++; } } }
static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cq *mcq = &cq->mcq; struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; struct mlx4_cqe *cqe; u16 index; u16 new_index, ring_index; u32 txbbs_skipped = 0; u32 cons_index = mcq->cons_index; int size = cq->size; u32 size_mask = ring->size_mask; struct mlx4_cqe *buf = cq->buf; if (!priv->port_up) return; index = cons_index & size_mask; cqe = &buf[index]; ring_index = ring->cons & size_mask; /* */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { /* */ rmb(); /* */ new_index = be16_to_cpu(cqe->wqe_index) & size_mask; do { txbbs_skipped += ring->last_nr_txbb; ring_index = (ring_index + ring->last_nr_txbb) & size_mask; /* */ ring->last_nr_txbb = mlx4_en_free_tx_desc( priv, ring, ring_index, !!((ring->cons + txbbs_skipped) & ring->size)); } while (ring_index != new_index); ++cons_index; index = cons_index & size_mask; cqe = &buf[index]; } /* */ mcq->cons_index = cons_index; mlx4_cq_set_ci(mcq); wmb(); ring->cons += txbbs_skipped; /* */ if (unlikely(ring->blocked)) { if ((u32) (ring->prod - ring->cons) <= ring->size - HEADROOM - MAX_DESC_TXBBS) { ring->blocked = 0; netif_tx_wake_queue(netdev_get_tx_queue(dev, cq->ring)); priv->port_stats.wake_queue++; } } }
static inline void dma_xmit_clean(struct net_device *dev, END_DEVICE *ei_local) { struct netdev_queue *txq; int cpu, clean_done = 0; u32 txd_free_idx; #if defined (CONFIG_RAETH_BQL) u32 bytes_sent_ge1 = 0; #if defined (CONFIG_PSEUDO_SUPPORT) u32 bytes_sent_ge2 = 0; #endif #endif spin_lock(&ei_local->page_lock); txd_free_idx = ei_local->txd_free_idx; while (clean_done < (NUM_TX_DESC-2)) { struct PDMA_txdesc *txd; struct sk_buff *skb; skb = ei_local->txd_buff[txd_free_idx]; if (!skb) break; txd = &ei_local->txd_ring[txd_free_idx]; /* check TXD not owned by DMA */ if (!(ACCESS_ONCE(txd->txd_info2) & TX2_DMA_DONE)) break; if (skb != (struct sk_buff *)0xFFFFFFFF) { #if defined (CONFIG_RAETH_BQL) #if defined (CONFIG_PSEUDO_SUPPORT) if (skb->dev == ei_local->PseudoDev) bytes_sent_ge2 += skb->len; else #endif bytes_sent_ge1 += skb->len; #endif dev_kfree_skb(skb); } ei_local->txd_buff[txd_free_idx] = NULL; txd_free_idx = (txd_free_idx + 1) % NUM_TX_DESC; clean_done++; } if (ei_local->txd_free_idx != txd_free_idx) ei_local->txd_free_idx = txd_free_idx; spin_unlock(&ei_local->page_lock); if (!clean_done) return; cpu = smp_processor_id(); if (netif_running(dev)) { txq = netdev_get_tx_queue(dev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge1); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #if defined (CONFIG_PSEUDO_SUPPORT) if (netif_running(ei_local->PseudoDev)) { txq = netdev_get_tx_queue(ei_local->PseudoDev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge2); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #endif }
static int mpodp_clean_tx_unlocked(struct mpodp_if_priv *priv, struct mpodp_txq *txq, unsigned budget) { struct net_device *netdev = priv->netdev; struct mpodp_tx *tx; unsigned int packets_completed = 0; unsigned int bytes_completed = 0; unsigned int worked = 0; union mppa_timestamp ts; uint32_t tx_done, first_tx_done, last_tx_done, tx_submitted, tx_size, tx_head; tx_submitted = atomic_read(&txq->submitted); tx_done = atomic_read(&txq->done); first_tx_done = tx_done; last_tx_done = first_tx_done; tx_size = txq->size; tx_head = atomic_read(&txq->head); if (!tx_head) { /* No carrier yet. Check if there are any buffers yet */ tx_head = readl(txq->head_addr); if (tx_head) { /* We now have buffers */ atomic_set(&txq->head, tx_head); if (netif_msg_link(priv)) netdev_info(netdev,"txq[%d] now has Tx (%u).\n", txq->id, tx_head); } return 0; } /* TX: 2nd step: update TX tail (DMA transfer completed) */ while (tx_done != tx_submitted && worked < budget) { if (!mpodp_tx_is_done(priv, txq, tx_done)) { /* DMA transfer not completed */ break; } if (netif_msg_tx_done(priv)) netdev_info(netdev, "txq[%d] tx[%d]: transfer done (head: %d submitted: %d done: %d)\n", txq->id, tx_done, atomic_read(&txq->head), tx_submitted, tx_done); /* get TX slot */ tx = &(txq->ring[tx_done]); /* free ressources */ unmap_skb(&priv->pdev->dev, tx->skb, tx); consume_skb(tx->skb); worked++; tx_done += 1; if (tx_done == tx_size) tx_done = 0; last_tx_done = tx_done; } /* write new TX tail */ atomic_set(&txq->done, tx_done); /* TX: 3rd step: free finished TX slot */ while (first_tx_done != last_tx_done) { if (netif_msg_tx_done(priv)) netdev_info(netdev, "txq[%d] tx[%d]: done (head: %d submitted: %d done: %d)\n", txq->id, first_tx_done, atomic_read(&txq->head), tx_submitted, tx_done); /* get TX slot */ tx = &(txq->ring[first_tx_done]); mppa_pcie_time_get(priv->tx_time, &ts); mppa_pcie_time_update(priv->tx_time, &tx->time, &ts); /* get stats */ packets_completed++; bytes_completed += tx->len; first_tx_done += 1; if (first_tx_done == tx_size) first_tx_done = 0; } if (!packets_completed) { goto out; } /* update stats */ netdev->stats.tx_bytes += bytes_completed; netdev->stats.tx_packets += packets_completed; netdev_tx_completed_queue(txq->txq, packets_completed, bytes_completed); netif_tx_wake_queue(txq->txq); out: return worked; }
static inline int netvsc_send_pkt( struct hv_netvsc_packet *packet, struct netvsc_device *net_device) { struct nvsp_message nvmsg; struct vmbus_channel *out_channel = packet->channel; u16 q_idx = packet->q_idx; struct net_device *ndev = net_device->ndev; u64 req_id; int ret; struct hv_page_buffer *pgbuf; nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; if (packet->is_data_pkt) { /* 0 is RMC_DATA; */ nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0; } else { /* 1 is RMC_CONTROL; */ nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1; } nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index = packet->send_buf_index; if (packet->send_buf_index == NETVSC_INVALID_INDEX) nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0; else nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = packet->total_data_buflen; if (packet->send_completion) req_id = (ulong)packet; else req_id = 0; if (out_channel->rescind) return -ENODEV; if (packet->page_buf_cnt) { pgbuf = packet->cp_partial ? packet->page_buf + packet->rmsg_pgcnt : packet->page_buf; ret = vmbus_sendpacket_pagebuffer(out_channel, pgbuf, packet->page_buf_cnt, &nvmsg, sizeof(struct nvsp_message), req_id); } else { ret = vmbus_sendpacket( out_channel, &nvmsg, sizeof(struct nvsp_message), req_id, VM_PKT_DATA_INBAND, VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); } if (ret == 0) { atomic_inc(&net_device->num_outstanding_sends); atomic_inc(&net_device->queue_sends[q_idx]); if (hv_ringbuf_avail_percent(&out_channel->outbound) < RING_AVAIL_PERCENT_LOWATER) { netif_tx_stop_queue(netdev_get_tx_queue( ndev, q_idx)); if (atomic_read(&net_device-> queue_sends[q_idx]) < 1) netif_tx_wake_queue(netdev_get_tx_queue( ndev, q_idx)); } } else if (ret == -EAGAIN) { netif_tx_stop_queue(netdev_get_tx_queue( ndev, q_idx)); if (atomic_read(&net_device->queue_sends[q_idx]) < 1) { netif_tx_wake_queue(netdev_get_tx_queue( ndev, q_idx)); ret = -ENOSPC; } } else { netdev_err(ndev, "Unable to send packet %p ret %d\n", packet, ret); } return ret; }
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq) { struct mlx5_cqe64 *cqe; struct mlx5e_sq *sq; u32 dma_fifo_cc; u32 nbytes; u16 npkts; u16 sqcc; int i; sq = container_of(cq, struct mlx5e_sq, cq); if (unlikely(test_bit(MLX5E_SQ_TX_TIMEOUT, &sq->state))) return false; npkts = 0; nbytes = 0; /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), * otherwise a cq overrun may occur */ sqcc = sq->cc; /* avoid dirtying sq cache line every cqe */ dma_fifo_cc = sq->dma_fifo_cc; cqe = mlx5e_get_cqe(cq); for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { u16 wqe_counter; bool last_wqe; if (!cqe) break; mlx5_cqwq_pop(&cq->wq); mlx5e_prefetch_cqe(cq); wqe_counter = be16_to_cpu(cqe->wqe_counter); do { struct mlx5e_tx_wqe_info *wi; struct sk_buff *skb; u16 ci; int j; last_wqe = (sqcc == wqe_counter); ci = sqcc & sq->wq.sz_m1; skb = sq->skb[ci]; wi = &sq->wqe_info[ci]; if (unlikely(!skb)) { /* nop */ sqcc++; continue; } if (unlikely(MLX5E_TX_HW_STAMP(sq->channel->priv, skb))) { struct skb_shared_hwtstamps hwts; mlx5e_fill_hwstamp(&sq->cq.channel->priv->tstamp, &hwts, get_cqe_ts(cqe)); skb_tstamp_tx(skb, &hwts); } for (j = 0; j < wi->num_dma; j++) { struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, dma_fifo_cc++); mlx5e_tx_dma_unmap(sq->pdev, dma); } npkts++; nbytes += wi->num_bytes; sqcc += wi->num_wqebbs; dev_kfree_skb(skb); } while (!last_wqe); cqe = mlx5e_get_cqe(cq); } mlx5_cqwq_update_db_record(&cq->wq); /* ensure cq space is freed before enabling more cqes */ wmb(); sq->dma_fifo_cc = dma_fifo_cc; sq->cc = sqcc; netdev_tx_completed_queue(sq->txq, npkts, nbytes); if (netif_tx_queue_stopped(sq->txq) && mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) && likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) { netif_tx_wake_queue(sq->txq); sq->stats.queue_wake++; } return (i == MLX5E_TX_CQ_POLL_BUDGET); }
static void ifb_ri_tasklet(unsigned long _txp) { struct ifb_q_private *txp = (struct ifb_q_private *)_txp; struct netdev_queue *txq; struct sk_buff *skb; txq = netdev_get_tx_queue(txp->dev, txp->txqnum); skb = skb_peek(&txp->tq); if (!skb) { if (!__netif_tx_trylock(txq)) goto resched; skb_queue_splice_tail_init(&txp->rq, &txp->tq); __netif_tx_unlock(txq); } while ((skb = __skb_dequeue(&txp->tq)) != NULL) { skb->tc_redirected = 0; skb->tc_skip_classify = 1; u64_stats_update_begin(&txp->tsync); txp->tx_packets++; txp->tx_bytes += skb->len; u64_stats_update_end(&txp->tsync); rcu_read_lock(); skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif); if (!skb->dev) { rcu_read_unlock(); dev_kfree_skb(skb); txp->dev->stats.tx_dropped++; if (skb_queue_len(&txp->tq) != 0) goto resched; break; } rcu_read_unlock(); skb->skb_iif = txp->dev->ifindex; if (!skb->tc_from_ingress) { dev_queue_xmit(skb); } else { skb_pull_rcsum(skb, skb->mac_len); netif_receive_skb(skb); } } if (__netif_tx_trylock(txq)) { skb = skb_peek(&txp->rq); if (!skb) { txp->tasklet_pending = 0; if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); } else { __netif_tx_unlock(txq); goto resched; } __netif_tx_unlock(txq); } else { resched: txp->tasklet_pending = 1; tasklet_schedule(&txp->ifb_tasklet); } }
int netvsc_send(struct hv_device *device, struct hv_netvsc_packet *packet, bool kick_q) { struct netvsc_device *net_device; int ret = 0; struct nvsp_message sendMessage; struct net_device *ndev; struct vmbus_channel *out_channel = NULL; u64 req_id; unsigned int section_index = NETVSC_INVALID_INDEX; u32 msg_size = 0; u16 q_idx = packet->q_idx; u32 vmbus_flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; net_device = get_outbound_net_device(device); if (!net_device) return -ENODEV; ndev = net_device->ndev; sendMessage.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; if (packet->is_data_pkt) { /* 0 is RMC_DATA; */ sendMessage.msg.v1_msg.send_rndis_pkt.channel_type = 0; } else { /* 1 is RMC_CONTROL; */ sendMessage.msg.v1_msg.send_rndis_pkt.channel_type = 1; } /* Attempt to send via sendbuf */ if (packet->total_data_buflen < net_device->send_section_size) { section_index = netvsc_get_next_send_section(net_device); if (section_index != NETVSC_INVALID_INDEX) { msg_size = netvsc_copy_to_send_buf(net_device, section_index, packet); packet->page_buf_cnt = 0; } } packet->send_buf_index = section_index; sendMessage.msg.v1_msg.send_rndis_pkt.send_buf_section_index = section_index; sendMessage.msg.v1_msg.send_rndis_pkt.send_buf_section_size = msg_size; if (packet->send_completion) req_id = (ulong)packet; else req_id = 0; out_channel = net_device->chn_table[packet->q_idx]; if (out_channel == NULL) out_channel = device->channel; packet->channel = out_channel; if (out_channel->rescind) return -ENODEV; if (packet->page_buf_cnt) { ret = vmbus_sendpacket_pagebuffer_ctl(out_channel, packet->page_buf, packet->page_buf_cnt, &sendMessage, sizeof(struct nvsp_message), req_id, vmbus_flags, kick_q); } else { ret = vmbus_sendpacket_ctl(out_channel, &sendMessage, sizeof(struct nvsp_message), req_id, VM_PKT_DATA_INBAND, vmbus_flags, kick_q); } if (ret == 0) { atomic_inc(&net_device->num_outstanding_sends); atomic_inc(&net_device->queue_sends[q_idx]); if (hv_ringbuf_avail_percent(&out_channel->outbound) < RING_AVAIL_PERCENT_LOWATER) { netif_tx_stop_queue(netdev_get_tx_queue( ndev, q_idx)); if (atomic_read(&net_device-> queue_sends[q_idx]) < 1) netif_tx_wake_queue(netdev_get_tx_queue( ndev, q_idx)); } } else if (ret == -EAGAIN) { netif_tx_stop_queue(netdev_get_tx_queue( ndev, q_idx)); if (atomic_read(&net_device->queue_sends[q_idx]) < 1) { netif_tx_wake_queue(netdev_get_tx_queue( ndev, q_idx)); ret = -ENOSPC; } } else { netdev_err(ndev, "Unable to send packet %p ret %d\n", packet, ret); } if (ret != 0 && section_index != NETVSC_INVALID_INDEX) netvsc_free_send_slot(net_device, section_index); return ret; }
static void fjes_tx_retry(struct net_device *netdev) { struct netdev_queue *queue = netdev_get_tx_queue(netdev, 0); netif_tx_wake_queue(queue); }
static inline void dma_xmit_clean(struct net_device *dev, END_DEVICE *ei_local) { struct netdev_queue *txq; int cpu, clean_done = 0; u32 cpu_ptr, dma_ptr, cpu_idx; #if defined (CONFIG_RAETH_BQL) u32 bytes_sent_ge1 = 0; #if defined (CONFIG_PSEUDO_SUPPORT) u32 bytes_sent_ge2 = 0; #endif #endif spin_lock(&ei_local->page_lock); cpu_ptr = sysRegRead(QTX_CRX_PTR); dma_ptr = sysRegRead(QTX_DRX_PTR); /* get current CPU TXD index */ cpu_idx = get_txd_offset(ei_local, cpu_ptr); while (cpu_ptr != dma_ptr) { struct QDMA_txdesc *txd; struct sk_buff *skb; txd = &ei_local->txd_pool[cpu_idx]; /* check TXD not owned by DMA */ if (!(ACCESS_ONCE(txd->txd_info3) & TX3_QDMA_OWN)) break; /* hold next TXD ptr */ cpu_ptr = ACCESS_ONCE(txd->txd_info2); /* release current TXD */ put_free_txd(ei_local, cpu_idx); /* get next TXD index */ cpu_idx = get_txd_offset(ei_local, cpu_ptr); /* free skb */ skb = ei_local->txd_buff[cpu_idx]; if (skb) { #if defined (CONFIG_RAETH_BQL) #if defined (CONFIG_PSEUDO_SUPPORT) if (skb->dev == ei_local->PseudoDev) bytes_sent_ge2 += skb->len; else #endif bytes_sent_ge1 += skb->len; #endif ei_local->txd_buff[cpu_idx] = NULL; dev_kfree_skb(skb); } clean_done++; /* prevent infinity loop when something wrong */ if (clean_done > (NUM_TX_DESC-4)) break; } if (clean_done) sysRegWrite(QTX_CRX_PTR, cpu_ptr); spin_unlock(&ei_local->page_lock); if (!clean_done) return; cpu = smp_processor_id(); if (netif_running(dev)) { txq = netdev_get_tx_queue(dev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge1); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #if defined (CONFIG_PSEUDO_SUPPORT) if (netif_running(ei_local->PseudoDev)) { txq = netdev_get_tx_queue(ei_local->PseudoDev, 0); __netif_tx_lock(txq, cpu); #if defined (CONFIG_RAETH_BQL) netdev_tx_completed_queue(txq, 0, bytes_sent_ge2); #endif if (netif_tx_queue_stopped(txq)) netif_tx_wake_queue(txq); __netif_tx_unlock(txq); } #endif }
u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, int index, u64 timestamp, int napi_mode) { struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; struct mlx4_en_rx_alloc frame = { .page = tx_info->page, .dma = tx_info->map0_dma, }; if (!mlx4_en_rx_recycle(ring->recycle_ring, &frame)) { dma_unmap_page(priv->ddev, tx_info->map0_dma, PAGE_SIZE, priv->dma_dir); put_page(tx_info->page); } return tx_info->nr_txbb; } int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) { struct mlx4_en_priv *priv = netdev_priv(dev); int cnt = 0; /* Skip last polled descriptor */ ring->cons += ring->last_nr_txbb; en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n", ring->cons, ring->prod); if ((u32) (ring->prod - ring->cons) > ring->size) { if (netif_msg_tx_err(priv)) en_warn(priv, "Tx consumer passed producer!\n"); return 0; } while (ring->cons != ring->prod) { ring->last_nr_txbb = ring->free_tx_desc(priv, ring, ring->cons & ring->size_mask, 0, 0 /* Non-NAPI caller */); ring->cons += ring->last_nr_txbb; cnt++; } if (ring->tx_queue) netdev_tx_reset_queue(ring->tx_queue); if (cnt) en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt); return cnt; } bool mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int napi_budget) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cq *mcq = &cq->mcq; struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->type][cq->ring]; struct mlx4_cqe *cqe; u16 index, ring_index, stamp_index; u32 txbbs_skipped = 0; u32 txbbs_stamp = 0; u32 cons_index = mcq->cons_index; int size = cq->size; u32 size_mask = ring->size_mask; struct mlx4_cqe *buf = cq->buf; u32 packets = 0; u32 bytes = 0; int factor = priv->cqe_factor; int done = 0; int budget = priv->tx_work_limit; u32 last_nr_txbb; u32 ring_cons; if (unlikely(!priv->port_up)) return true; netdev_txq_bql_complete_prefetchw(ring->tx_queue); index = cons_index & size_mask; cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; last_nr_txbb = READ_ONCE(ring->last_nr_txbb); ring_cons = READ_ONCE(ring->cons); ring_index = ring_cons & size_mask; stamp_index = ring_index; /* Process all completed CQEs */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size) && (done < budget)) { u16 new_index; /* * make sure we read the CQE after we read the * ownership bit */ dma_rmb(); if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_ERROR)) { struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe; en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n", cqe_err->vendor_err_syndrome, cqe_err->syndrome); } /* Skip over last polled CQE */ new_index = be16_to_cpu(cqe->wqe_index) & size_mask; do { u64 timestamp = 0; txbbs_skipped += last_nr_txbb; ring_index = (ring_index + last_nr_txbb) & size_mask; if (unlikely(ring->tx_info[ring_index].ts_requested)) timestamp = mlx4_en_get_cqe_ts(cqe); /* free next descriptor */ last_nr_txbb = ring->free_tx_desc( priv, ring, ring_index, timestamp, napi_budget); mlx4_en_stamp_wqe(priv, ring, stamp_index, !!((ring_cons + txbbs_stamp) & ring->size)); stamp_index = ring_index; txbbs_stamp = txbbs_skipped; packets++; bytes += ring->tx_info[ring_index].nr_bytes; } while ((++done < budget) && (ring_index != new_index)); ++cons_index; index = cons_index & size_mask; cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor; } /* * To prevent CQ overflow we first update CQ consumer and only then * the ring consumer. */ mcq->cons_index = cons_index; mlx4_cq_set_ci(mcq); wmb(); /* we want to dirty this cache line once */ WRITE_ONCE(ring->last_nr_txbb, last_nr_txbb); WRITE_ONCE(ring->cons, ring_cons + txbbs_skipped); if (cq->type == TX_XDP) return done < budget; netdev_tx_completed_queue(ring->tx_queue, packets, bytes); /* Wakeup Tx queue if this stopped, and ring is not full. */ if (netif_tx_queue_stopped(ring->tx_queue) && !mlx4_en_is_tx_ring_full(ring)) { netif_tx_wake_queue(ring->tx_queue); ring->wake_queue++; } return done < budget; } void mlx4_en_tx_irq(struct mlx4_cq *mcq) { struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq); struct mlx4_en_priv *priv = netdev_priv(cq->dev); if (likely(priv->port_up)) napi_schedule_irqoff(&cq->napi); else mlx4_en_arm_cq(priv, cq); } /* TX CQ polling - called by NAPI */ int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget) { struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi); struct net_device *dev = cq->dev; struct mlx4_en_priv *priv = netdev_priv(dev); bool clean_complete; clean_complete = mlx4_en_process_tx_cq(dev, cq, budget); if (!clean_complete) return budget; napi_complete(napi); mlx4_en_arm_cq(priv, cq); return 0; } static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, u32 index, unsigned int desc_size) { u32 copy = (ring->size - index) << LOG_TXBB_SIZE; int i; for (i = desc_size - copy - 4; i >= 0; i -= 4) { if ((i & (TXBB_SIZE - 1)) == 0) wmb(); *((u32 *) (ring->buf + i)) = *((u32 *) (ring->bounce_buf + copy + i)); } for (i = copy - 4; i >= 4 ; i -= 4) { if ((i & (TXBB_SIZE - 1)) == 0) wmb(); *((u32 *)(ring->buf + (index << LOG_TXBB_SIZE) + i)) = *((u32 *) (ring->bounce_buf + i)); } /* Return real descriptor location */ return ring->buf + (index << LOG_TXBB_SIZE); }
/** * nfp_net_tx_complete() - Handled completed TX packets * @tx_ring: TX ring structure * * Return: Number of completed TX descriptors */ static void nfp_net_tx_complete(struct nfp_net_tx_ring *tx_ring) { struct nfp_net_r_vector *r_vec = tx_ring->r_vec; struct nfp_net *nn = r_vec->nfp_net; const struct skb_frag_struct *frag; struct netdev_queue *nd_q; u32 done_pkts = 0, done_bytes = 0; struct sk_buff *skb; int todo, nr_frags; u32 qcp_rd_p; int fidx; int idx; /* Work out how many descriptors have been transmitted */ qcp_rd_p = nfp_qcp_rd_ptr_read(tx_ring->qcp_q); if (qcp_rd_p == tx_ring->qcp_rd_p) return; if (qcp_rd_p > tx_ring->qcp_rd_p) todo = qcp_rd_p - tx_ring->qcp_rd_p; else todo = qcp_rd_p + tx_ring->cnt - tx_ring->qcp_rd_p; while (todo--) { idx = tx_ring->rd_p % tx_ring->cnt; tx_ring->rd_p++; skb = tx_ring->txbufs[idx].skb; if (!skb) continue; nr_frags = skb_shinfo(skb)->nr_frags; fidx = tx_ring->txbufs[idx].fidx; if (fidx == -1) { /* unmap head */ dma_unmap_single(&nn->pdev->dev, tx_ring->txbufs[idx].dma_addr, skb_headlen(skb), DMA_TO_DEVICE); done_pkts += tx_ring->txbufs[idx].pkt_cnt; done_bytes += tx_ring->txbufs[idx].real_len; } else { /* unmap fragment */ frag = &skb_shinfo(skb)->frags[fidx]; dma_unmap_page(&nn->pdev->dev, tx_ring->txbufs[idx].dma_addr, skb_frag_size(frag), DMA_TO_DEVICE); } /* check for last gather fragment */ if (fidx == nr_frags - 1) dev_kfree_skb_any(skb); tx_ring->txbufs[idx].dma_addr = 0; tx_ring->txbufs[idx].skb = NULL; tx_ring->txbufs[idx].fidx = -2; } tx_ring->qcp_rd_p = qcp_rd_p; u64_stats_update_begin(&r_vec->tx_sync); r_vec->tx_bytes += done_bytes; r_vec->tx_pkts += done_pkts; u64_stats_update_end(&r_vec->tx_sync); nd_q = netdev_get_tx_queue(nn->netdev, tx_ring->idx); netdev_tx_completed_queue(nd_q, done_pkts, done_bytes); if (nfp_net_tx_ring_should_wake(tx_ring)) { /* Make sure TX thread will see updated tx_ring->rd_p */ smp_mb(); if (unlikely(netif_tx_queue_stopped(nd_q))) netif_tx_wake_queue(nd_q); } WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt, "TX ring corruption rd_p=%u wr_p=%u cnt=%u\n", tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt); }
int mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cq *mcq = &cq->mcq; struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; struct mlx4_cqe *cqe; u16 index; u16 new_index, ring_index; u32 txbbs_skipped = 0; u32 cons_index = mcq->cons_index; int size = cq->size; u32 size_mask = ring->size_mask; struct mlx4_cqe *buf = cq->buf; int factor = priv->cqe_factor; int done = 0; index = cons_index & size_mask; cqe = &buf[(index << factor) + factor]; ring_index = ring->cons & size_mask; /* Process all completed CQEs */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size) && done < budget) { /* * make sure we read the CQE after we read the * ownership bit */ rmb(); /* Skip over last polled CQE */ new_index = be16_to_cpu(cqe->wqe_index) & size_mask; do { txbbs_skipped += ring->last_nr_txbb; ring_index = (ring_index + ring->last_nr_txbb) & size_mask; /* free next descriptor */ ring->last_nr_txbb = mlx4_en_free_tx_desc( priv, ring, ring_index, !!((ring->cons + txbbs_skipped) & ring->size)); } while ((++done < budget) && ring_index != new_index); ++cons_index; index = cons_index & size_mask; cqe = &buf[(index << factor) + factor]; } /* * To prevent CQ overflow we first update CQ consumer and only then * the ring consumer. */ mcq->cons_index = cons_index; mlx4_cq_set_ci(mcq); wmb(); ring->cons += txbbs_skipped; atomic_sub(txbbs_skipped, &ring->inflight); /* Wakeup Tx queue if this ring stopped it */ if (unlikely(ring->blocked && txbbs_skipped > 0)) { ring->blocked = 0; #ifndef __VMKERNEL_MLX4_EN_TX_HASH__ netif_tx_wake_queue(netdev_get_tx_queue(dev, cq->ring)); #else netif_tx_wake_queue(netdev_get_tx_queue(dev, ring->reported_index)); #endif /* NOT __VMKERNEL_MLX4_EN_TX_HASH__ */ priv->port_stats.wake_queue++; } return done; }
static void netvsc_send_completion(struct netvsc_device *net_device, struct hv_device *device, struct vmpacket_descriptor *packet) { struct nvsp_message *nvsp_packet; struct hv_netvsc_packet *nvsc_packet; struct net_device *ndev; u32 send_index; ndev = net_device->ndev; nvsp_packet = (struct nvsp_message *)((unsigned long)packet + (packet->offset8 << 3)); if ((nvsp_packet->hdr.msg_type == NVSP_MSG_TYPE_INIT_COMPLETE) || (nvsp_packet->hdr.msg_type == NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE) || (nvsp_packet->hdr.msg_type == NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE) || (nvsp_packet->hdr.msg_type == NVSP_MSG5_TYPE_SUBCHANNEL)) { /* Copy the response back */ memcpy(&net_device->channel_init_pkt, nvsp_packet, sizeof(struct nvsp_message)); complete(&net_device->channel_init_wait); } else if (nvsp_packet->hdr.msg_type == NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE) { int num_outstanding_sends; u16 q_idx = 0; struct vmbus_channel *channel = device->channel; int queue_sends; /* Get the send context */ nvsc_packet = (struct hv_netvsc_packet *)(unsigned long) packet->trans_id; /* Notify the layer above us */ if (nvsc_packet) { send_index = nvsc_packet->send_buf_index; if (send_index != NETVSC_INVALID_INDEX) netvsc_free_send_slot(net_device, send_index); q_idx = nvsc_packet->q_idx; channel = nvsc_packet->channel; nvsc_packet->send_completion(nvsc_packet-> send_completion_ctx); } num_outstanding_sends = atomic_dec_return(&net_device->num_outstanding_sends); queue_sends = atomic_dec_return(&net_device-> queue_sends[q_idx]); if (net_device->destroy && num_outstanding_sends == 0) wake_up(&net_device->wait_drain); if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) && !net_device->start_remove && (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER || queue_sends < 1)) netif_tx_wake_queue(netdev_get_tx_queue( ndev, q_idx)); } else { netdev_err(ndev, "Unknown send completion packet type- " "%d received!!\n", nvsp_packet->hdr.msg_type); } }
/**============================================================================ @brief hdd_tx_fetch_packet_cbk() - Callback function invoked by TL to fetch a packet for transmission. @param vosContext : [in] pointer to VOS context @param staId : [in] Station for which TL is requesting a pkt @param ac : [in] access category requested by TL @param pVosPacket : [out] pointer to VOS packet packet pointer @param pPktMetaInfo : [out] pointer to meta info for the pkt @return : VOS_STATUS_E_EMPTY if no packets to transmit : VOS_STATUS_E_FAILURE if any errors encountered : VOS_STATUS_SUCCESS otherwise ===========================================================================*/ VOS_STATUS hdd_tx_fetch_packet_cbk( v_VOID_t *vosContext, v_U8_t *pStaId, WLANTL_ACEnumType ac, vos_pkt_t **ppVosPacket, WLANTL_MetaInfoType *pPktMetaInfo ) { VOS_STATUS status = VOS_STATUS_E_FAILURE; hdd_adapter_t *pAdapter = NULL; hdd_context_t *pHddCtx = NULL; hdd_list_node_t *anchor = NULL; skb_list_node_t *pktNode = NULL; struct sk_buff *skb = NULL; vos_pkt_t *pVosPacket = NULL; v_MACADDR_t* pDestMacAddress = NULL; v_TIME_t timestamp; WLANTL_ACEnumType newAc; v_SIZE_t size = 0; tANI_U8 acAdmitted, i; //Sanity check on inputs if ( ( NULL == vosContext ) || ( NULL == pStaId ) || ( NULL == ppVosPacket ) || ( NULL == pPktMetaInfo ) ) { VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR,"%s: Null Params being passed", __FUNCTION__); return VOS_STATUS_E_FAILURE; } //Get the HDD context. pHddCtx = (hdd_context_t *)vos_get_context( VOS_MODULE_ID_HDD, vosContext ); if(pHddCtx == NULL) { VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR,"%s: HDD adapter context is Null", __FUNCTION__); return VOS_STATUS_E_FAILURE; } pAdapter = pHddCtx->sta_to_adapter[*pStaId]; if( NULL == pAdapter ) { VOS_ASSERT(0); return VOS_STATUS_E_FAILURE; } ++pAdapter->hdd_stats.hddTxRxStats.txFetched; *ppVosPacket = NULL; //Make sure the AC being asked for is sane if( ac >= WLANTL_MAX_AC || ac < 0) { VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR,"%s: Invalid AC %d passed by TL", __FUNCTION__, ac); return VOS_STATUS_E_FAILURE; } ++pAdapter->hdd_stats.hddTxRxStats.txFetchedAC[ac]; #ifdef HDD_WMM_DEBUG VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL,"%s: AC %d passed by TL", __FUNCTION__, ac); #endif // HDD_WMM_DEBUG // We find an AC with packets // or we determine we have no more packets to send // HDD is not allowed to change AC. // has this AC been admitted? or // To allow EAPOL packets when not authenticated if (unlikely((0==pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcAccessAllowed) && (WLAN_HDD_GET_STATION_CTX_PTR(pAdapter))->conn_info.uIsAuthenticated)) { ++pAdapter->hdd_stats.hddTxRxStats.txFetchEmpty; #ifdef HDD_WMM_DEBUG VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL, "%s: no packets pending", __FUNCTION__); #endif // HDD_WMM_DEBUG return VOS_STATUS_E_FAILURE; } // do we have any packets pending in this AC? hdd_list_size( &pAdapter->wmm_tx_queue[ac], &size ); if( size > 0 ) { // yes, so process it #ifdef HDD_WMM_DEBUG VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL, "%s: AC %d has packets pending", __FUNCTION__, ac); #endif // HDD_WMM_DEBUG } else { ++pAdapter->hdd_stats.hddTxRxStats.txFetchEmpty; #ifdef HDD_WMM_DEBUG VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL, "%s: no packets pending", __FUNCTION__); #endif // HDD_WMM_DEBUG return VOS_STATUS_E_FAILURE; } //Get the vos packet. I don't want to dequeue and enqueue again if we are out of VOS resources //This simplifies the locking and unlocking of Tx queue status = vos_pkt_wrap_data_packet( &pVosPacket, VOS_PKT_TYPE_TX_802_3_DATA, NULL, //OS Pkt is not being passed hdd_tx_low_resource_cbk, pAdapter ); if (status == VOS_STATUS_E_ALREADY || status == VOS_STATUS_E_RESOURCES) { //Remember VOS is in a low resource situation pAdapter->isVosOutOfResource = VOS_TRUE; ++pAdapter->hdd_stats.hddTxRxStats.txFetchLowResources; VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,"%s: VOSS in Low Resource scenario", __FUNCTION__); //TL will now think we have no more packets in this AC return VOS_STATUS_E_FAILURE; } //Remove the packet from the queue spin_lock_bh(&pAdapter->wmm_tx_queue[ac].lock); status = hdd_list_remove_front( &pAdapter->wmm_tx_queue[ac], &anchor ); spin_unlock_bh(&pAdapter->wmm_tx_queue[ac].lock); if(VOS_STATUS_SUCCESS == status) { //If success then we got a valid packet from some AC pktNode = list_entry(anchor, skb_list_node_t, anchor); skb = pktNode->skb; } else { ++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeueError; VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN, "%s: Error in de-queuing " "skb from Tx queue status = %d", __FUNCTION__, status ); vos_pkt_return_packet(pVosPacket); return VOS_STATUS_E_FAILURE; } //Attach skb to VOS packet. status = vos_pkt_set_os_packet( pVosPacket, skb ); if (status != VOS_STATUS_SUCCESS) { VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,"%s: Error attaching skb", __FUNCTION__); vos_pkt_return_packet(pVosPacket); ++pAdapter->stats.tx_dropped; ++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeueError; kfree_skb(skb); return VOS_STATUS_E_FAILURE; } //Just being paranoid. To be removed later if(pVosPacket == NULL) { VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,"%s: VOS packet returned by VOSS is NULL", __FUNCTION__); ++pAdapter->stats.tx_dropped; ++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeueError; kfree_skb(skb); return VOS_STATUS_E_FAILURE; } //Return VOS packet to TL; *ppVosPacket = pVosPacket; //Fill out the meta information needed by TL //FIXME This timestamp is really the time stamp of wrap_data_packet vos_pkt_get_timestamp( pVosPacket, ×tamp ); pPktMetaInfo->usTimeStamp = (v_U16_t)timestamp; if(pAdapter->sessionCtx.station.conn_info.uIsAuthenticated == VOS_TRUE) pPktMetaInfo->ucIsEapol = 0; else pPktMetaInfo->ucIsEapol = hdd_IsEAPOLPacket( pVosPacket ) ? 1 : 0; #ifdef FEATURE_WLAN_WAPI // Override usIsEapol value when its zero for WAPI case pPktMetaInfo->ucIsWai = hdd_IsWAIPacket( pVosPacket ) ? 1 : 0; #endif /* FEATURE_WLAN_WAPI */ if ((HDD_WMM_USER_MODE_NO_QOS == pHddCtx->cfg_ini->WmmMode) || (!pAdapter->hddWmmStatus.wmmQap)) { // either we don't want QoS or the AP doesn't support QoS pPktMetaInfo->ucUP = 0; pPktMetaInfo->ucTID = 0; } else { /* 1. Check if ACM is set for this AC * 2. If set, check if this AC had already admitted * 3. If not already admitted, downgrade the UP to next best UP */ if(!pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcAccessRequired || pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcTspecValid) { pPktMetaInfo->ucUP = pktNode->userPriority; pPktMetaInfo->ucTID = pPktMetaInfo->ucUP; } else { //Downgrade the UP acAdmitted = pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcTspecValid; newAc = WLANTL_AC_BK; for (i=ac-1; i>0; i--) { if (pAdapter->hddWmmStatus.wmmAcStatus[i].wmmAcAccessRequired == 0) { newAc = i; break; } } pPktMetaInfo->ucUP = hddWmmAcToHighestUp[newAc]; pPktMetaInfo->ucTID = pPktMetaInfo->ucUP; VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_INFO_LOW,"Downgrading UP %d to UP %d ", pktNode->userPriority, pPktMetaInfo->ucUP); } } pPktMetaInfo->ucType = 0; //FIXME Don't know what this is pPktMetaInfo->ucDisableFrmXtl = 0; //802.3 frame so we need to xlate if ( 1 < size ) { pPktMetaInfo->bMorePackets = 1; //HDD has more packets to send } else { pPktMetaInfo->bMorePackets = 0; } //Extract the destination address from ethernet frame pDestMacAddress = (v_MACADDR_t*)skb->data; pPktMetaInfo->ucBcast = vos_is_macaddr_broadcast( pDestMacAddress ) ? 1 : 0; pPktMetaInfo->ucMcast = vos_is_macaddr_group( pDestMacAddress ) ? 1 : 0; // if we are in a backpressure situation see if we can turn the hose back on if ( (pAdapter->isTxSuspended[ac]) && (size <= HDD_TX_QUEUE_LOW_WATER_MARK) ) { ++pAdapter->hdd_stats.hddTxRxStats.txFetchDePressured; ++pAdapter->hdd_stats.hddTxRxStats.txFetchDePressuredAC[ac]; VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN, "%s: TX queue[%d] re-enabled", __FUNCTION__, ac); pAdapter->isTxSuspended[ac] = VOS_FALSE; netif_tx_wake_queue(netdev_get_tx_queue(pAdapter->dev, skb_get_queue_mapping(skb) )); } // We're giving the packet to TL so consider it transmitted from // a statistics perspective. We account for it here instead of // when the packet is returned for two reasons. First, TL will // manipulate the skb to the point where the len field is not // accurate, leading to inaccurate byte counts if we account for // it later. Second, TL does not provide any feedback as to // whether or not the packet was successfully sent over the air, // so the packet counts will be the same regardless of where we // account for them pAdapter->stats.tx_bytes += skb->len; ++pAdapter->stats.tx_packets; ++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeued; ++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeuedAC[ac]; if(pHddCtx->cfg_ini->thermalMitigationEnable) { if(mutex_lock_interruptible(&pHddCtx->tmInfo.tmOperationLock)) { VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR, "%s: Tm Lock fail", __FUNCTION__); return VOS_STATUS_E_FAILURE; } if(WLAN_HDD_TM_LEVEL_1 < pHddCtx->tmInfo.currentTmLevel) { if(0 == pHddCtx->tmInfo.txFrameCount) { /* Just recovered from sleep timeout */ pHddCtx->tmInfo.lastOpenTs = timestamp; } if(((timestamp - pHddCtx->tmInfo.lastOpenTs) > (pHddCtx->tmInfo.tmAction.txOperationDuration / 10)) && (pHddCtx->tmInfo.txFrameCount >= pHddCtx->tmInfo.tmAction.txBlockFrameCountThreshold)) { spin_lock(&pAdapter->wmm_tx_queue[ac].lock); /* During TX open duration, TX frame count is larger than threshold * Block TX during Sleep time */ netif_tx_stop_all_queues(pAdapter->dev); spin_unlock(&pAdapter->wmm_tx_queue[ac].lock); pHddCtx->tmInfo.lastblockTs = timestamp; if(VOS_TIMER_STATE_STOPPED == vos_timer_getCurrentState(&pHddCtx->tmInfo.txSleepTimer)) { vos_timer_start(&pHddCtx->tmInfo.txSleepTimer, pHddCtx->tmInfo.tmAction.txSleepDuration); } } else if(((timestamp - pHddCtx->tmInfo.lastOpenTs) > (pHddCtx->tmInfo.tmAction.txOperationDuration / 10)) && (pHddCtx->tmInfo.txFrameCount < pHddCtx->tmInfo.tmAction.txBlockFrameCountThreshold)) { /* During TX open duration, TX frame count is less than threshold * Reset count and timestamp to prepare next cycle */ pHddCtx->tmInfo.lastOpenTs = timestamp; pHddCtx->tmInfo.txFrameCount = 0; } else { /* Do Nothing */ } pHddCtx->tmInfo.txFrameCount++; } mutex_unlock(&pHddCtx->tmInfo.tmOperationLock); } #ifdef HDD_WMM_DEBUG VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL,"%s: Valid VOS PKT returned to TL", __FUNCTION__); #endif // HDD_WMM_DEBUG return status; }
void xenvif_wake_queue(struct xenvif_queue *queue) { struct net_device *dev = queue->vif->dev; unsigned int id = queue->id; netif_tx_wake_queue(netdev_get_tx_queue(dev, id)); }
static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cq *mcq = &cq->mcq; struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring]; struct mlx4_cqe *cqe; u16 index; u16 new_index, ring_index; u32 txbbs_skipped = 0; u32 cons_index = mcq->cons_index; int size = cq->size; u32 size_mask = ring->size_mask; struct mlx4_cqe *buf = cq->buf; if (!priv->port_up) return; index = cons_index & size_mask; cqe = &buf[index]; ring_index = ring->cons & size_mask; /* Process all completed CQEs */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { /* * make sure we read the CQE after we read the * ownership bit */ rmb(); /* Skip over last polled CQE */ new_index = be16_to_cpu(cqe->wqe_index) & size_mask; do { txbbs_skipped += ring->last_nr_txbb; ring_index = (ring_index + ring->last_nr_txbb) & size_mask; /* free next descriptor */ ring->last_nr_txbb = mlx4_en_free_tx_desc( priv, ring, ring_index, !!((ring->cons + txbbs_skipped) & ring->size)); } while (ring_index != new_index); ++cons_index; index = cons_index & size_mask; cqe = &buf[index]; } /* * To prevent CQ overflow we first update CQ consumer and only then * the ring consumer. */ mcq->cons_index = cons_index; mlx4_cq_set_ci(mcq); wmb(); ring->cons += txbbs_skipped; /* Wakeup Tx queue if this ring stopped it */ if (unlikely(ring->blocked)) { if ((u32) (ring->prod - ring->cons) <= ring->size - HEADROOM - MAX_DESC_TXBBS) { ring->blocked = 0; netif_tx_wake_queue(netdev_get_tx_queue(dev, cq->ring)); priv->port_stats.wake_queue++; } } }