void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
{
unsigned fill_level;
struct efx_nic *efx = tx_queue->efx;
EFX_BUG_ON_PARANOID(index > tx_queue->ptr_mask);
efx_dequeue_buffers(tx_queue, index);
/* See if we need to restart the netif queue. This barrier
* separates the update of read_count from the test of the
* queue state. */
smp_mb();
if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&
likely(efx->port_enabled)) {
fill_level = tx_queue->insert_count - tx_queue->read_count;
if (fill_level < EFX_TXQ_THRESHOLD(efx)) {
EFX_BUG_ON_PARANOID(!efx_dev_registered(efx));
netif_tx_wake_queue(tx_queue->core_txq);
}
}
/* Check whether the hardware queue is now empty */
if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {
tx_queue->old_write_count = ACCESS_ONCE(tx_queue->write_count);
if (tx_queue->read_count == tx_queue->old_write_count) {
smp_mb();
tx_queue->empty_read_count =
tx_queue->read_count | EFX_EMPTY_COUNT_VALID;
}
}
}
/*
* Packet send completion callback handler.
*
* It either frees the buffer directly or forwards it to another
* completion callback which checks conditions, updates statistics,
* wakes up stalled traffic queue if required, and then frees the buffer.
*/
int mwifiex_write_data_complete(struct mwifiex_adapter *adapter,
struct sk_buff *skb, int aggr, int status)
{
struct mwifiex_private *priv;
struct mwifiex_txinfo *tx_info;
struct netdev_queue *txq;
int index;
if (!skb)
return 0;
tx_info = MWIFIEX_SKB_TXCB(skb);
priv = mwifiex_get_priv_by_id(adapter, tx_info->bss_num,
tx_info->bss_type);
if (!priv)
goto done;
if (adapter->iface_type == MWIFIEX_USB)
adapter->data_sent = false;
mwifiex_set_trans_start(priv->netdev);
if (!status) {
priv->stats.tx_packets++;
priv->stats.tx_bytes += skb->len;
if (priv->tx_timeout_cnt)
priv->tx_timeout_cnt = 0;
} else {
priv->stats.tx_errors++;
}
if (tx_info->flags & MWIFIEX_BUF_FLAG_BRIDGED_PKT)
atomic_dec_return(&adapter->pending_bridged_pkts);
if (aggr)
/* For skb_aggr, do not wake up tx queue */
goto done;
atomic_dec(&adapter->tx_pending);
index = mwifiex_1d_to_wmm_queue[skb->priority];
if (atomic_dec_return(&priv->wmm_tx_pending[index]) < LOW_TX_PENDING) {
txq = netdev_get_tx_queue(priv->netdev, index);
if (netif_tx_queue_stopped(txq)) {
netif_tx_wake_queue(txq);
dev_dbg(adapter->dev, "wake queue: %d\n", index);
}
}
done:
dev_kfree_skb_any(skb);
return 0;
}
/*
* This function wakes up all queues in net_device
*/
void mwifiex_wake_up_net_dev_queue(struct net_device *netdev,
struct mwifiex_adapter *adapter)
{
unsigned long dev_queue_flags;
unsigned int i;
spin_lock_irqsave(&adapter->queue_lock, dev_queue_flags);
for (i = 0; i < netdev->num_tx_queues; i++) {
struct netdev_queue *txq = netdev_get_tx_queue(netdev, i);
if (netif_tx_queue_stopped(txq))
netif_tx_wake_queue(txq);
}
spin_unlock_irqrestore(&adapter->queue_lock, dev_queue_flags);
}
static int xlgmac_tx_poll(struct xlgmac_channel *channel)
{
struct xlgmac_pdata *pdata = channel->pdata;
struct xlgmac_ring *ring = channel->tx_ring;
struct net_device *netdev = pdata->netdev;
unsigned int tx_packets = 0, tx_bytes = 0;
struct xlgmac_desc_data *desc_data;
struct xlgmac_dma_desc *dma_desc;
struct xlgmac_desc_ops *desc_ops;
struct xlgmac_hw_ops *hw_ops;
struct netdev_queue *txq;
int processed = 0;
unsigned int cur;
desc_ops = &pdata->desc_ops;
hw_ops = &pdata->hw_ops;
/* Nothing to do if there isn't a Tx ring for this channel */
if (!ring)
return 0;
cur = ring->cur;
/* Be sure we get ring->cur before accessing descriptor data */
smp_rmb();
txq = netdev_get_tx_queue(netdev, channel->queue_index);
while ((processed < XLGMAC_TX_DESC_MAX_PROC) &&
(ring->dirty != cur)) {
desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty);
dma_desc = desc_data->dma_desc;
if (!hw_ops->tx_complete(dma_desc))
break;
/* Make sure descriptor fields are read after reading
* the OWN bit
*/
dma_rmb();
if (netif_msg_tx_done(pdata))
xlgmac_dump_tx_desc(pdata, ring, ring->dirty, 1, 0);
if (hw_ops->is_last_desc(dma_desc)) {
tx_packets += desc_data->tx.packets;
tx_bytes += desc_data->tx.bytes;
}
/* Free the SKB and reset the descriptor for re-use */
desc_ops->unmap_desc_data(pdata, desc_data);
hw_ops->tx_desc_reset(desc_data);
processed++;
ring->dirty++;
}
if (!processed)
return 0;
netdev_tx_completed_queue(txq, tx_packets, tx_bytes);
if ((ring->tx.queue_stopped == 1) &&
(xlgmac_tx_avail_desc(ring) > XLGMAC_TX_DESC_MIN_FREE)) {
ring->tx.queue_stopped = 0;
netif_tx_wake_queue(txq);
}
XLGMAC_PR("processed=%d\n", processed);
return processed;
}
static bool mlx4_en_process_tx_cq(struct ether *dev,
struct mlx4_en_cq *cq)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring];
struct mlx4_cqe *cqe;
uint16_t index;
uint16_t new_index, ring_index, stamp_index;
uint32_t txbbs_skipped = 0;
uint32_t txbbs_stamp = 0;
uint32_t cons_index = mcq->cons_index;
int size = cq->size;
uint32_t size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
uint32_t packets = 0;
uint32_t bytes = 0;
int factor = priv->cqe_factor;
uint64_t timestamp = 0;
int done = 0;
int budget = priv->tx_work_limit;
uint32_t last_nr_txbb;
uint32_t ring_cons;
if (!priv->port_up)
return true;
#if 0 // AKAROS_PORT
netdev_txq_bql_complete_prefetchw(ring->tx_queue);
#endif
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb);
ring_cons = ACCESS_ONCE(ring->cons);
ring_index = ring_cons & size_mask;
stamp_index = ring_index;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size) && (done < budget)) {
/*
* make sure we read the CQE after we read the
* ownership bit
*/
bus_rmb();
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe;
en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
cqe_err->vendor_err_syndrome,
cqe_err->syndrome);
}
/* Skip over last polled CQE */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
txbbs_skipped += last_nr_txbb;
ring_index = (ring_index + last_nr_txbb) & size_mask;
if (ring->tx_info[ring_index].ts_requested)
timestamp = mlx4_en_get_cqe_ts(cqe);
/* free next descriptor */
last_nr_txbb = mlx4_en_free_tx_desc(
priv, ring, ring_index,
!!((ring_cons + txbbs_skipped) &
ring->size), timestamp);
mlx4_en_stamp_wqe(priv, ring, stamp_index,
!!((ring_cons + txbbs_stamp) &
ring->size));
stamp_index = ring_index;
txbbs_stamp = txbbs_skipped;
packets++;
bytes += ring->tx_info[ring_index].nr_bytes;
} while ((++done < budget) && (ring_index != new_index));
++cons_index;
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
}
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
/* we want to dirty this cache line once */
ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb;
ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped;
#if 0 // AKAROS_PORT
netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
/*
* Wakeup Tx queue if this stopped, and at least 1 packet
* was completed
*/
if (netif_tx_queue_stopped(ring->tx_queue) && txbbs_skipped > 0) {
netif_tx_wake_queue(ring->tx_queue);
ring->wake_queue++;
}
#endif
return done < budget;
}
static inline int netvsc_send_pkt(
struct hv_netvsc_packet *packet,
struct netvsc_device *net_device)
{
struct nvsp_message nvmsg;
struct vmbus_channel *out_channel = packet->channel;
u16 q_idx = packet->q_idx;
struct net_device *ndev = net_device->ndev;
u64 req_id;
int ret;
struct hv_page_buffer *pgbuf;
u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound);
nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
if (packet->is_data_pkt) {
/* 0 is RMC_DATA; */
nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0;
} else {
/* 1 is RMC_CONTROL; */
nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1;
}
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index =
packet->send_buf_index;
if (packet->send_buf_index == NETVSC_INVALID_INDEX)
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0;
else
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size =
packet->total_data_buflen;
if (packet->send_completion)
req_id = (ulong)packet;
else
req_id = 0;
if (out_channel->rescind)
return -ENODEV;
/*
* It is possible that once we successfully place this packet
* on the ringbuffer, we may stop the queue. In that case, we want
* to notify the host independent of the xmit_more flag. We don't
* need to be precise here; in the worst case we may signal the host
* unnecessarily.
*/
if (ring_avail < (RING_AVAIL_PERCENT_LOWATER + 1))
packet->xmit_more = false;
if (packet->page_buf_cnt) {
pgbuf = packet->cp_partial ? packet->page_buf +
packet->rmsg_pgcnt : packet->page_buf;
ret = vmbus_sendpacket_pagebuffer_ctl(out_channel,
pgbuf,
packet->page_buf_cnt,
&nvmsg,
sizeof(struct nvsp_message),
req_id,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED,
!packet->xmit_more);
} else {
ret = vmbus_sendpacket_ctl(out_channel, &nvmsg,
sizeof(struct nvsp_message),
req_id,
VM_PKT_DATA_INBAND,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED,
!packet->xmit_more);
}
if (ret == 0) {
atomic_inc(&net_device->num_outstanding_sends);
atomic_inc(&net_device->queue_sends[q_idx]);
if (ring_avail < RING_AVAIL_PERCENT_LOWATER) {
netif_tx_stop_queue(netdev_get_tx_queue(ndev, q_idx));
if (atomic_read(&net_device->
queue_sends[q_idx]) < 1)
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
}
} else if (ret == -EAGAIN) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->queue_sends[q_idx]) < 1) {
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
ret = -ENOSPC;
}
} else {
netdev_err(ndev, "Unable to send packet %p ret %d\n",
packet, ret);
}
return ret;
}
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq)
{
struct mlx5e_sq *sq;
u32 dma_fifo_cc;
u32 nbytes;
u16 npkts;
u16 sqcc;
int i;
/* avoid accessing cq (dma coherent memory) if not needed */
if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags))
return false;
sq = container_of(cq, struct mlx5e_sq, cq);
npkts = 0;
nbytes = 0;
/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
* otherwise a cq overrun may occur
*/
sqcc = sq->cc;
/* avoid dirtying sq cache line every cqe */
dma_fifo_cc = sq->dma_fifo_cc;
for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
struct mlx5_cqe64 *cqe;
u16 wqe_counter;
bool last_wqe;
cqe = mlx5e_get_cqe(cq);
if (!cqe)
break;
mlx5_cqwq_pop(&cq->wq);
wqe_counter = be16_to_cpu(cqe->wqe_counter);
do {
struct sk_buff *skb;
u16 ci;
int j;
last_wqe = (sqcc == wqe_counter);
ci = sqcc & sq->wq.sz_m1;
skb = sq->skb[ci];
if (unlikely(!skb)) { /* nop */
sq->stats.nop++;
sqcc++;
continue;
}
for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) {
dma_addr_t addr;
u32 size;
mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size);
dma_fifo_cc++;
dma_unmap_single(sq->pdev, addr, size,
DMA_TO_DEVICE);
}
npkts++;
nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes;
sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs;
dev_kfree_skb(skb);
} while (!last_wqe);
}
mlx5_cqwq_update_db_record(&cq->wq);
/* ensure cq space is freed before enabling more cqes */
wmb();
sq->dma_fifo_cc = dma_fifo_cc;
sq->cc = sqcc;
netdev_tx_completed_queue(sq->txq, npkts, nbytes);
if (netif_tx_queue_stopped(sq->txq) &&
mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) &&
likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) {
netif_tx_wake_queue(sq->txq);
sq->stats.wake++;
}
if (i == MLX5E_TX_CQ_POLL_BUDGET) {
set_bit(MLX5E_CQ_HAS_CQES, &cq->flags);
return true;
}
return false;
}
static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring];
struct mlx4_cqe *cqe = cq->buf;
u16 index;
u16 new_index;
u32 txbbs_skipped = 0;
u32 cq_last_sav;
/* index always points to the first TXBB of the last polled descriptor */
index = ring->cons & ring->size_mask;
new_index = be16_to_cpu(cqe->wqe_index) & ring->size_mask;
if (index == new_index)
return;
if (!priv->port_up)
return;
/*
* We use a two-stage loop:
* - the first samples the HW-updated CQE
* - the second frees TXBBs until the last sample
* This lets us amortize CQE cache misses, while still polling the CQ
* until is quiescent.
*/
cq_last_sav = mcq->cons_index;
do {
do {
/* Skip over last polled CQE */
index = (index + ring->last_nr_txbb) & ring->size_mask;
txbbs_skipped += ring->last_nr_txbb;
/* Poll next CQE */
ring->last_nr_txbb = mlx4_en_free_tx_desc(
priv, ring, index,
!!((ring->cons + txbbs_skipped) &
ring->size));
++mcq->cons_index;
} while (index != new_index);
new_index = be16_to_cpu(cqe->wqe_index) & ring->size_mask;
} while (index != new_index);
AVG_PERF_COUNTER(priv->pstats.tx_coal_avg,
(u32) (mcq->cons_index - cq_last_sav));
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mlx4_cq_set_ci(mcq);
wmb();
ring->cons += txbbs_skipped;
/* Wakeup Tx queue if this ring stopped it */
if (unlikely(ring->blocked)) {
if ((u32) (ring->prod - ring->cons) <=
ring->size - HEADROOM - MAX_DESC_TXBBS) {
ring->blocked = 0;
netif_tx_wake_queue(netdev_get_tx_queue(dev, cq->ring));
priv->port_stats.wake_queue++;
}
}
}
static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring];
struct mlx4_cqe *cqe;
u16 index;
u16 new_index, ring_index;
u32 txbbs_skipped = 0;
u32 cons_index = mcq->cons_index;
int size = cq->size;
u32 size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
if (!priv->port_up)
return;
index = cons_index & size_mask;
cqe = &buf[index];
ring_index = ring->cons & size_mask;
/* */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size)) {
/*
*/
rmb();
/* */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
txbbs_skipped += ring->last_nr_txbb;
ring_index = (ring_index + ring->last_nr_txbb) & size_mask;
/* */
ring->last_nr_txbb = mlx4_en_free_tx_desc(
priv, ring, ring_index,
!!((ring->cons + txbbs_skipped) &
ring->size));
} while (ring_index != new_index);
++cons_index;
index = cons_index & size_mask;
cqe = &buf[index];
}
/*
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
ring->cons += txbbs_skipped;
/* */
if (unlikely(ring->blocked)) {
if ((u32) (ring->prod - ring->cons) <=
ring->size - HEADROOM - MAX_DESC_TXBBS) {
ring->blocked = 0;
netif_tx_wake_queue(netdev_get_tx_queue(dev, cq->ring));
priv->port_stats.wake_queue++;
}
}
}
static inline void
dma_xmit_clean(struct net_device *dev, END_DEVICE *ei_local)
{
struct netdev_queue *txq;
int cpu, clean_done = 0;
u32 txd_free_idx;
#if defined (CONFIG_RAETH_BQL)
u32 bytes_sent_ge1 = 0;
#if defined (CONFIG_PSEUDO_SUPPORT)
u32 bytes_sent_ge2 = 0;
#endif
#endif
spin_lock(&ei_local->page_lock);
txd_free_idx = ei_local->txd_free_idx;
while (clean_done < (NUM_TX_DESC-2)) {
struct PDMA_txdesc *txd;
struct sk_buff *skb;
skb = ei_local->txd_buff[txd_free_idx];
if (!skb)
break;
txd = &ei_local->txd_ring[txd_free_idx];
/* check TXD not owned by DMA */
if (!(ACCESS_ONCE(txd->txd_info2) & TX2_DMA_DONE))
break;
if (skb != (struct sk_buff *)0xFFFFFFFF) {
#if defined (CONFIG_RAETH_BQL)
#if defined (CONFIG_PSEUDO_SUPPORT)
if (skb->dev == ei_local->PseudoDev)
bytes_sent_ge2 += skb->len;
else
#endif
bytes_sent_ge1 += skb->len;
#endif
dev_kfree_skb(skb);
}
ei_local->txd_buff[txd_free_idx] = NULL;
txd_free_idx = (txd_free_idx + 1) % NUM_TX_DESC;
clean_done++;
}
if (ei_local->txd_free_idx != txd_free_idx)
ei_local->txd_free_idx = txd_free_idx;
spin_unlock(&ei_local->page_lock);
if (!clean_done)
return;
cpu = smp_processor_id();
if (netif_running(dev)) {
txq = netdev_get_tx_queue(dev, 0);
__netif_tx_lock(txq, cpu);
#if defined (CONFIG_RAETH_BQL)
netdev_tx_completed_queue(txq, 0, bytes_sent_ge1);
#endif
if (netif_tx_queue_stopped(txq))
netif_tx_wake_queue(txq);
__netif_tx_unlock(txq);
}
#if defined (CONFIG_PSEUDO_SUPPORT)
if (netif_running(ei_local->PseudoDev)) {
txq = netdev_get_tx_queue(ei_local->PseudoDev, 0);
__netif_tx_lock(txq, cpu);
#if defined (CONFIG_RAETH_BQL)
netdev_tx_completed_queue(txq, 0, bytes_sent_ge2);
#endif
if (netif_tx_queue_stopped(txq))
netif_tx_wake_queue(txq);
__netif_tx_unlock(txq);
}
#endif
}
static int mpodp_clean_tx_unlocked(struct mpodp_if_priv *priv,
struct mpodp_txq *txq, unsigned budget)
{
struct net_device *netdev = priv->netdev;
struct mpodp_tx *tx;
unsigned int packets_completed = 0;
unsigned int bytes_completed = 0;
unsigned int worked = 0;
union mppa_timestamp ts;
uint32_t tx_done, first_tx_done, last_tx_done, tx_submitted,
tx_size, tx_head;
tx_submitted = atomic_read(&txq->submitted);
tx_done = atomic_read(&txq->done);
first_tx_done = tx_done;
last_tx_done = first_tx_done;
tx_size = txq->size;
tx_head = atomic_read(&txq->head);
if (!tx_head) {
/* No carrier yet. Check if there are any buffers yet */
tx_head = readl(txq->head_addr);
if (tx_head) {
/* We now have buffers */
atomic_set(&txq->head, tx_head);
if (netif_msg_link(priv))
netdev_info(netdev,"txq[%d] now has Tx (%u).\n",
txq->id, tx_head);
}
return 0;
}
/* TX: 2nd step: update TX tail (DMA transfer completed) */
while (tx_done != tx_submitted && worked < budget) {
if (!mpodp_tx_is_done(priv, txq, tx_done)) {
/* DMA transfer not completed */
break;
}
if (netif_msg_tx_done(priv))
netdev_info(netdev,
"txq[%d] tx[%d]: transfer done (head: %d submitted: %d done: %d)\n",
txq->id, tx_done, atomic_read(&txq->head),
tx_submitted, tx_done);
/* get TX slot */
tx = &(txq->ring[tx_done]);
/* free ressources */
unmap_skb(&priv->pdev->dev, tx->skb, tx);
consume_skb(tx->skb);
worked++;
tx_done += 1;
if (tx_done == tx_size)
tx_done = 0;
last_tx_done = tx_done;
}
/* write new TX tail */
atomic_set(&txq->done, tx_done);
/* TX: 3rd step: free finished TX slot */
while (first_tx_done != last_tx_done) {
if (netif_msg_tx_done(priv))
netdev_info(netdev,
"txq[%d] tx[%d]: done (head: %d submitted: %d done: %d)\n",
txq->id, first_tx_done, atomic_read(&txq->head),
tx_submitted, tx_done);
/* get TX slot */
tx = &(txq->ring[first_tx_done]);
mppa_pcie_time_get(priv->tx_time, &ts);
mppa_pcie_time_update(priv->tx_time, &tx->time, &ts);
/* get stats */
packets_completed++;
bytes_completed += tx->len;
first_tx_done += 1;
if (first_tx_done == tx_size)
first_tx_done = 0;
}
if (!packets_completed) {
goto out;
}
/* update stats */
netdev->stats.tx_bytes += bytes_completed;
netdev->stats.tx_packets += packets_completed;
netdev_tx_completed_queue(txq->txq, packets_completed, bytes_completed);
netif_tx_wake_queue(txq->txq);
out:
return worked;
}
static inline int netvsc_send_pkt(
struct hv_netvsc_packet *packet,
struct netvsc_device *net_device)
{
struct nvsp_message nvmsg;
struct vmbus_channel *out_channel = packet->channel;
u16 q_idx = packet->q_idx;
struct net_device *ndev = net_device->ndev;
u64 req_id;
int ret;
struct hv_page_buffer *pgbuf;
nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
if (packet->is_data_pkt) {
/* 0 is RMC_DATA; */
nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0;
} else {
/* 1 is RMC_CONTROL; */
nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1;
}
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index =
packet->send_buf_index;
if (packet->send_buf_index == NETVSC_INVALID_INDEX)
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0;
else
nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size =
packet->total_data_buflen;
if (packet->send_completion)
req_id = (ulong)packet;
else
req_id = 0;
if (out_channel->rescind)
return -ENODEV;
if (packet->page_buf_cnt) {
pgbuf = packet->cp_partial ? packet->page_buf +
packet->rmsg_pgcnt : packet->page_buf;
ret = vmbus_sendpacket_pagebuffer(out_channel,
pgbuf,
packet->page_buf_cnt,
&nvmsg,
sizeof(struct nvsp_message),
req_id);
} else {
ret = vmbus_sendpacket(
out_channel, &nvmsg,
sizeof(struct nvsp_message),
req_id,
VM_PKT_DATA_INBAND,
VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
}
if (ret == 0) {
atomic_inc(&net_device->num_outstanding_sends);
atomic_inc(&net_device->queue_sends[q_idx]);
if (hv_ringbuf_avail_percent(&out_channel->outbound) <
RING_AVAIL_PERCENT_LOWATER) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->
queue_sends[q_idx]) < 1)
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
}
} else if (ret == -EAGAIN) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->queue_sends[q_idx]) < 1) {
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
ret = -ENOSPC;
}
} else {
netdev_err(ndev, "Unable to send packet %p ret %d\n",
packet, ret);
}
return ret;
}
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq)
{
struct mlx5_cqe64 *cqe;
struct mlx5e_sq *sq;
u32 dma_fifo_cc;
u32 nbytes;
u16 npkts;
u16 sqcc;
int i;
sq = container_of(cq, struct mlx5e_sq, cq);
if (unlikely(test_bit(MLX5E_SQ_TX_TIMEOUT, &sq->state)))
return false;
npkts = 0;
nbytes = 0;
/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
* otherwise a cq overrun may occur */
sqcc = sq->cc;
/* avoid dirtying sq cache line every cqe */
dma_fifo_cc = sq->dma_fifo_cc;
cqe = mlx5e_get_cqe(cq);
for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
u16 wqe_counter;
bool last_wqe;
if (!cqe)
break;
mlx5_cqwq_pop(&cq->wq);
mlx5e_prefetch_cqe(cq);
wqe_counter = be16_to_cpu(cqe->wqe_counter);
do {
struct mlx5e_tx_wqe_info *wi;
struct sk_buff *skb;
u16 ci;
int j;
last_wqe = (sqcc == wqe_counter);
ci = sqcc & sq->wq.sz_m1;
skb = sq->skb[ci];
wi = &sq->wqe_info[ci];
if (unlikely(!skb)) { /* nop */
sqcc++;
continue;
}
if (unlikely(MLX5E_TX_HW_STAMP(sq->channel->priv,
skb))) {
struct skb_shared_hwtstamps hwts;
mlx5e_fill_hwstamp(&sq->cq.channel->priv->tstamp,
&hwts, get_cqe_ts(cqe));
skb_tstamp_tx(skb, &hwts);
}
for (j = 0; j < wi->num_dma; j++) {
struct mlx5e_sq_dma *dma =
mlx5e_dma_get(sq, dma_fifo_cc++);
mlx5e_tx_dma_unmap(sq->pdev, dma);
}
npkts++;
nbytes += wi->num_bytes;
sqcc += wi->num_wqebbs;
dev_kfree_skb(skb);
} while (!last_wqe);
cqe = mlx5e_get_cqe(cq);
}
mlx5_cqwq_update_db_record(&cq->wq);
/* ensure cq space is freed before enabling more cqes */
wmb();
sq->dma_fifo_cc = dma_fifo_cc;
sq->cc = sqcc;
netdev_tx_completed_queue(sq->txq, npkts, nbytes);
if (netif_tx_queue_stopped(sq->txq) &&
mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) &&
likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) {
netif_tx_wake_queue(sq->txq);
sq->stats.queue_wake++;
}
return (i == MLX5E_TX_CQ_POLL_BUDGET);
}
static void ifb_ri_tasklet(unsigned long _txp)
{
struct ifb_q_private *txp = (struct ifb_q_private *)_txp;
struct netdev_queue *txq;
struct sk_buff *skb;
txq = netdev_get_tx_queue(txp->dev, txp->txqnum);
skb = skb_peek(&txp->tq);
if (!skb) {
if (!__netif_tx_trylock(txq))
goto resched;
skb_queue_splice_tail_init(&txp->rq, &txp->tq);
__netif_tx_unlock(txq);
}
while ((skb = __skb_dequeue(&txp->tq)) != NULL) {
skb->tc_redirected = 0;
skb->tc_skip_classify = 1;
u64_stats_update_begin(&txp->tsync);
txp->tx_packets++;
txp->tx_bytes += skb->len;
u64_stats_update_end(&txp->tsync);
rcu_read_lock();
skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif);
if (!skb->dev) {
rcu_read_unlock();
dev_kfree_skb(skb);
txp->dev->stats.tx_dropped++;
if (skb_queue_len(&txp->tq) != 0)
goto resched;
break;
}
rcu_read_unlock();
skb->skb_iif = txp->dev->ifindex;
if (!skb->tc_from_ingress) {
dev_queue_xmit(skb);
} else {
skb_pull_rcsum(skb, skb->mac_len);
netif_receive_skb(skb);
}
}
if (__netif_tx_trylock(txq)) {
skb = skb_peek(&txp->rq);
if (!skb) {
txp->tasklet_pending = 0;
if (netif_tx_queue_stopped(txq))
netif_tx_wake_queue(txq);
} else {
__netif_tx_unlock(txq);
goto resched;
}
__netif_tx_unlock(txq);
} else {
resched:
txp->tasklet_pending = 1;
tasklet_schedule(&txp->ifb_tasklet);
}
}
int netvsc_send(struct hv_device *device,
struct hv_netvsc_packet *packet, bool kick_q)
{
struct netvsc_device *net_device;
int ret = 0;
struct nvsp_message sendMessage;
struct net_device *ndev;
struct vmbus_channel *out_channel = NULL;
u64 req_id;
unsigned int section_index = NETVSC_INVALID_INDEX;
u32 msg_size = 0;
u16 q_idx = packet->q_idx;
u32 vmbus_flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
net_device = get_outbound_net_device(device);
if (!net_device)
return -ENODEV;
ndev = net_device->ndev;
sendMessage.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
if (packet->is_data_pkt) {
/* 0 is RMC_DATA; */
sendMessage.msg.v1_msg.send_rndis_pkt.channel_type = 0;
} else {
/* 1 is RMC_CONTROL; */
sendMessage.msg.v1_msg.send_rndis_pkt.channel_type = 1;
}
/* Attempt to send via sendbuf */
if (packet->total_data_buflen < net_device->send_section_size) {
section_index = netvsc_get_next_send_section(net_device);
if (section_index != NETVSC_INVALID_INDEX) {
msg_size = netvsc_copy_to_send_buf(net_device,
section_index,
packet);
packet->page_buf_cnt = 0;
}
}
packet->send_buf_index = section_index;
sendMessage.msg.v1_msg.send_rndis_pkt.send_buf_section_index =
section_index;
sendMessage.msg.v1_msg.send_rndis_pkt.send_buf_section_size = msg_size;
if (packet->send_completion)
req_id = (ulong)packet;
else
req_id = 0;
out_channel = net_device->chn_table[packet->q_idx];
if (out_channel == NULL)
out_channel = device->channel;
packet->channel = out_channel;
if (out_channel->rescind)
return -ENODEV;
if (packet->page_buf_cnt) {
ret = vmbus_sendpacket_pagebuffer_ctl(out_channel,
packet->page_buf,
packet->page_buf_cnt,
&sendMessage,
sizeof(struct nvsp_message),
req_id,
vmbus_flags,
kick_q);
} else {
ret = vmbus_sendpacket_ctl(out_channel, &sendMessage,
sizeof(struct nvsp_message),
req_id,
VM_PKT_DATA_INBAND,
vmbus_flags,
kick_q);
}
if (ret == 0) {
atomic_inc(&net_device->num_outstanding_sends);
atomic_inc(&net_device->queue_sends[q_idx]);
if (hv_ringbuf_avail_percent(&out_channel->outbound) <
RING_AVAIL_PERCENT_LOWATER) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->
queue_sends[q_idx]) < 1)
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
}
} else if (ret == -EAGAIN) {
netif_tx_stop_queue(netdev_get_tx_queue(
ndev, q_idx));
if (atomic_read(&net_device->queue_sends[q_idx]) < 1) {
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
ret = -ENOSPC;
}
} else {
netdev_err(ndev, "Unable to send packet %p ret %d\n",
packet, ret);
}
if (ret != 0 && section_index != NETVSC_INVALID_INDEX)
netvsc_free_send_slot(net_device, section_index);
return ret;
}
static void fjes_tx_retry(struct net_device *netdev)
{
struct netdev_queue *queue = netdev_get_tx_queue(netdev, 0);
netif_tx_wake_queue(queue);
}
static inline void
dma_xmit_clean(struct net_device *dev, END_DEVICE *ei_local)
{
struct netdev_queue *txq;
int cpu, clean_done = 0;
u32 cpu_ptr, dma_ptr, cpu_idx;
#if defined (CONFIG_RAETH_BQL)
u32 bytes_sent_ge1 = 0;
#if defined (CONFIG_PSEUDO_SUPPORT)
u32 bytes_sent_ge2 = 0;
#endif
#endif
spin_lock(&ei_local->page_lock);
cpu_ptr = sysRegRead(QTX_CRX_PTR);
dma_ptr = sysRegRead(QTX_DRX_PTR);
/* get current CPU TXD index */
cpu_idx = get_txd_offset(ei_local, cpu_ptr);
while (cpu_ptr != dma_ptr) {
struct QDMA_txdesc *txd;
struct sk_buff *skb;
txd = &ei_local->txd_pool[cpu_idx];
/* check TXD not owned by DMA */
if (!(ACCESS_ONCE(txd->txd_info3) & TX3_QDMA_OWN))
break;
/* hold next TXD ptr */
cpu_ptr = ACCESS_ONCE(txd->txd_info2);
/* release current TXD */
put_free_txd(ei_local, cpu_idx);
/* get next TXD index */
cpu_idx = get_txd_offset(ei_local, cpu_ptr);
/* free skb */
skb = ei_local->txd_buff[cpu_idx];
if (skb) {
#if defined (CONFIG_RAETH_BQL)
#if defined (CONFIG_PSEUDO_SUPPORT)
if (skb->dev == ei_local->PseudoDev)
bytes_sent_ge2 += skb->len;
else
#endif
bytes_sent_ge1 += skb->len;
#endif
ei_local->txd_buff[cpu_idx] = NULL;
dev_kfree_skb(skb);
}
clean_done++;
/* prevent infinity loop when something wrong */
if (clean_done > (NUM_TX_DESC-4))
break;
}
if (clean_done)
sysRegWrite(QTX_CRX_PTR, cpu_ptr);
spin_unlock(&ei_local->page_lock);
if (!clean_done)
return;
cpu = smp_processor_id();
if (netif_running(dev)) {
txq = netdev_get_tx_queue(dev, 0);
__netif_tx_lock(txq, cpu);
#if defined (CONFIG_RAETH_BQL)
netdev_tx_completed_queue(txq, 0, bytes_sent_ge1);
#endif
if (netif_tx_queue_stopped(txq))
netif_tx_wake_queue(txq);
__netif_tx_unlock(txq);
}
#if defined (CONFIG_PSEUDO_SUPPORT)
if (netif_running(ei_local->PseudoDev)) {
txq = netdev_get_tx_queue(ei_local->PseudoDev, 0);
__netif_tx_lock(txq, cpu);
#if defined (CONFIG_RAETH_BQL)
netdev_tx_completed_queue(txq, 0, bytes_sent_ge2);
#endif
if (netif_tx_queue_stopped(txq))
netif_tx_wake_queue(txq);
__netif_tx_unlock(txq);
}
#endif
}
u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring,
int index, u64 timestamp,
int napi_mode)
{
struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
struct mlx4_en_rx_alloc frame = {
.page = tx_info->page,
.dma = tx_info->map0_dma,
};
if (!mlx4_en_rx_recycle(ring->recycle_ring, &frame)) {
dma_unmap_page(priv->ddev, tx_info->map0_dma,
PAGE_SIZE, priv->dma_dir);
put_page(tx_info->page);
}
return tx_info->nr_txbb;
}
int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
int cnt = 0;
/* Skip last polled descriptor */
ring->cons += ring->last_nr_txbb;
en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
ring->cons, ring->prod);
if ((u32) (ring->prod - ring->cons) > ring->size) {
if (netif_msg_tx_err(priv))
en_warn(priv, "Tx consumer passed producer!\n");
return 0;
}
while (ring->cons != ring->prod) {
ring->last_nr_txbb = ring->free_tx_desc(priv, ring,
ring->cons & ring->size_mask,
0, 0 /* Non-NAPI caller */);
ring->cons += ring->last_nr_txbb;
cnt++;
}
if (ring->tx_queue)
netdev_tx_reset_queue(ring->tx_queue);
if (cnt)
en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt);
return cnt;
}
bool mlx4_en_process_tx_cq(struct net_device *dev,
struct mlx4_en_cq *cq, int napi_budget)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->type][cq->ring];
struct mlx4_cqe *cqe;
u16 index, ring_index, stamp_index;
u32 txbbs_skipped = 0;
u32 txbbs_stamp = 0;
u32 cons_index = mcq->cons_index;
int size = cq->size;
u32 size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
u32 packets = 0;
u32 bytes = 0;
int factor = priv->cqe_factor;
int done = 0;
int budget = priv->tx_work_limit;
u32 last_nr_txbb;
u32 ring_cons;
if (unlikely(!priv->port_up))
return true;
netdev_txq_bql_complete_prefetchw(ring->tx_queue);
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
last_nr_txbb = READ_ONCE(ring->last_nr_txbb);
ring_cons = READ_ONCE(ring->cons);
ring_index = ring_cons & size_mask;
stamp_index = ring_index;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size) && (done < budget)) {
u16 new_index;
/*
* make sure we read the CQE after we read the
* ownership bit
*/
dma_rmb();
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe;
en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
cqe_err->vendor_err_syndrome,
cqe_err->syndrome);
}
/* Skip over last polled CQE */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
u64 timestamp = 0;
txbbs_skipped += last_nr_txbb;
ring_index = (ring_index + last_nr_txbb) & size_mask;
if (unlikely(ring->tx_info[ring_index].ts_requested))
timestamp = mlx4_en_get_cqe_ts(cqe);
/* free next descriptor */
last_nr_txbb = ring->free_tx_desc(
priv, ring, ring_index,
timestamp, napi_budget);
mlx4_en_stamp_wqe(priv, ring, stamp_index,
!!((ring_cons + txbbs_stamp) &
ring->size));
stamp_index = ring_index;
txbbs_stamp = txbbs_skipped;
packets++;
bytes += ring->tx_info[ring_index].nr_bytes;
} while ((++done < budget) && (ring_index != new_index));
++cons_index;
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
}
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
/* we want to dirty this cache line once */
WRITE_ONCE(ring->last_nr_txbb, last_nr_txbb);
WRITE_ONCE(ring->cons, ring_cons + txbbs_skipped);
if (cq->type == TX_XDP)
return done < budget;
netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
/* Wakeup Tx queue if this stopped, and ring is not full.
*/
if (netif_tx_queue_stopped(ring->tx_queue) &&
!mlx4_en_is_tx_ring_full(ring)) {
netif_tx_wake_queue(ring->tx_queue);
ring->wake_queue++;
}
return done < budget;
}
void mlx4_en_tx_irq(struct mlx4_cq *mcq)
{
struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
struct mlx4_en_priv *priv = netdev_priv(cq->dev);
if (likely(priv->port_up))
napi_schedule_irqoff(&cq->napi);
else
mlx4_en_arm_cq(priv, cq);
}
/* TX CQ polling - called by NAPI */
int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget)
{
struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
struct net_device *dev = cq->dev;
struct mlx4_en_priv *priv = netdev_priv(dev);
bool clean_complete;
clean_complete = mlx4_en_process_tx_cq(dev, cq, budget);
if (!clean_complete)
return budget;
napi_complete(napi);
mlx4_en_arm_cq(priv, cq);
return 0;
}
static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring,
u32 index,
unsigned int desc_size)
{
u32 copy = (ring->size - index) << LOG_TXBB_SIZE;
int i;
for (i = desc_size - copy - 4; i >= 0; i -= 4) {
if ((i & (TXBB_SIZE - 1)) == 0)
wmb();
*((u32 *) (ring->buf + i)) =
*((u32 *) (ring->bounce_buf + copy + i));
}
for (i = copy - 4; i >= 4 ; i -= 4) {
if ((i & (TXBB_SIZE - 1)) == 0)
wmb();
*((u32 *)(ring->buf + (index << LOG_TXBB_SIZE) + i)) =
*((u32 *) (ring->bounce_buf + i));
}
/* Return real descriptor location */
return ring->buf + (index << LOG_TXBB_SIZE);
}
/**
* nfp_net_tx_complete() - Handled completed TX packets
* @tx_ring: TX ring structure
*
* Return: Number of completed TX descriptors
*/
static void nfp_net_tx_complete(struct nfp_net_tx_ring *tx_ring)
{
struct nfp_net_r_vector *r_vec = tx_ring->r_vec;
struct nfp_net *nn = r_vec->nfp_net;
const struct skb_frag_struct *frag;
struct netdev_queue *nd_q;
u32 done_pkts = 0, done_bytes = 0;
struct sk_buff *skb;
int todo, nr_frags;
u32 qcp_rd_p;
int fidx;
int idx;
/* Work out how many descriptors have been transmitted */
qcp_rd_p = nfp_qcp_rd_ptr_read(tx_ring->qcp_q);
if (qcp_rd_p == tx_ring->qcp_rd_p)
return;
if (qcp_rd_p > tx_ring->qcp_rd_p)
todo = qcp_rd_p - tx_ring->qcp_rd_p;
else
todo = qcp_rd_p + tx_ring->cnt - tx_ring->qcp_rd_p;
while (todo--) {
idx = tx_ring->rd_p % tx_ring->cnt;
tx_ring->rd_p++;
skb = tx_ring->txbufs[idx].skb;
if (!skb)
continue;
nr_frags = skb_shinfo(skb)->nr_frags;
fidx = tx_ring->txbufs[idx].fidx;
if (fidx == -1) {
/* unmap head */
dma_unmap_single(&nn->pdev->dev,
tx_ring->txbufs[idx].dma_addr,
skb_headlen(skb), DMA_TO_DEVICE);
done_pkts += tx_ring->txbufs[idx].pkt_cnt;
done_bytes += tx_ring->txbufs[idx].real_len;
} else {
/* unmap fragment */
frag = &skb_shinfo(skb)->frags[fidx];
dma_unmap_page(&nn->pdev->dev,
tx_ring->txbufs[idx].dma_addr,
skb_frag_size(frag), DMA_TO_DEVICE);
}
/* check for last gather fragment */
if (fidx == nr_frags - 1)
dev_kfree_skb_any(skb);
tx_ring->txbufs[idx].dma_addr = 0;
tx_ring->txbufs[idx].skb = NULL;
tx_ring->txbufs[idx].fidx = -2;
}
tx_ring->qcp_rd_p = qcp_rd_p;
u64_stats_update_begin(&r_vec->tx_sync);
r_vec->tx_bytes += done_bytes;
r_vec->tx_pkts += done_pkts;
u64_stats_update_end(&r_vec->tx_sync);
nd_q = netdev_get_tx_queue(nn->netdev, tx_ring->idx);
netdev_tx_completed_queue(nd_q, done_pkts, done_bytes);
if (nfp_net_tx_ring_should_wake(tx_ring)) {
/* Make sure TX thread will see updated tx_ring->rd_p */
smp_mb();
if (unlikely(netif_tx_queue_stopped(nd_q)))
netif_tx_wake_queue(nd_q);
}
WARN_ONCE(tx_ring->wr_p - tx_ring->rd_p > tx_ring->cnt,
"TX ring corruption rd_p=%u wr_p=%u cnt=%u\n",
tx_ring->rd_p, tx_ring->wr_p, tx_ring->cnt);
}
int mlx4_en_process_tx_cq(struct net_device *dev,
struct mlx4_en_cq *cq,
int budget)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring];
struct mlx4_cqe *cqe;
u16 index;
u16 new_index, ring_index;
u32 txbbs_skipped = 0;
u32 cons_index = mcq->cons_index;
int size = cq->size;
u32 size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
int factor = priv->cqe_factor;
int done = 0;
index = cons_index & size_mask;
cqe = &buf[(index << factor) + factor];
ring_index = ring->cons & size_mask;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size) && done < budget) {
/*
* make sure we read the CQE after we read the
* ownership bit
*/
rmb();
/* Skip over last polled CQE */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
txbbs_skipped += ring->last_nr_txbb;
ring_index = (ring_index + ring->last_nr_txbb) & size_mask;
/* free next descriptor */
ring->last_nr_txbb = mlx4_en_free_tx_desc(
priv, ring, ring_index,
!!((ring->cons + txbbs_skipped) &
ring->size));
} while ((++done < budget) && ring_index != new_index);
++cons_index;
index = cons_index & size_mask;
cqe = &buf[(index << factor) + factor];
}
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
ring->cons += txbbs_skipped;
atomic_sub(txbbs_skipped, &ring->inflight);
/* Wakeup Tx queue if this ring stopped it */
if (unlikely(ring->blocked && txbbs_skipped > 0)) {
ring->blocked = 0;
#ifndef __VMKERNEL_MLX4_EN_TX_HASH__
netif_tx_wake_queue(netdev_get_tx_queue(dev, cq->ring));
#else
netif_tx_wake_queue(netdev_get_tx_queue(dev, ring->reported_index));
#endif /* NOT __VMKERNEL_MLX4_EN_TX_HASH__ */
priv->port_stats.wake_queue++;
}
return done;
}
static void netvsc_send_completion(struct netvsc_device *net_device,
struct hv_device *device,
struct vmpacket_descriptor *packet)
{
struct nvsp_message *nvsp_packet;
struct hv_netvsc_packet *nvsc_packet;
struct net_device *ndev;
u32 send_index;
ndev = net_device->ndev;
nvsp_packet = (struct nvsp_message *)((unsigned long)packet +
(packet->offset8 << 3));
if ((nvsp_packet->hdr.msg_type == NVSP_MSG_TYPE_INIT_COMPLETE) ||
(nvsp_packet->hdr.msg_type ==
NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE) ||
(nvsp_packet->hdr.msg_type ==
NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE) ||
(nvsp_packet->hdr.msg_type ==
NVSP_MSG5_TYPE_SUBCHANNEL)) {
/* Copy the response back */
memcpy(&net_device->channel_init_pkt, nvsp_packet,
sizeof(struct nvsp_message));
complete(&net_device->channel_init_wait);
} else if (nvsp_packet->hdr.msg_type ==
NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE) {
int num_outstanding_sends;
u16 q_idx = 0;
struct vmbus_channel *channel = device->channel;
int queue_sends;
/* Get the send context */
nvsc_packet = (struct hv_netvsc_packet *)(unsigned long)
packet->trans_id;
/* Notify the layer above us */
if (nvsc_packet) {
send_index = nvsc_packet->send_buf_index;
if (send_index != NETVSC_INVALID_INDEX)
netvsc_free_send_slot(net_device, send_index);
q_idx = nvsc_packet->q_idx;
channel = nvsc_packet->channel;
nvsc_packet->send_completion(nvsc_packet->
send_completion_ctx);
}
num_outstanding_sends =
atomic_dec_return(&net_device->num_outstanding_sends);
queue_sends = atomic_dec_return(&net_device->
queue_sends[q_idx]);
if (net_device->destroy && num_outstanding_sends == 0)
wake_up(&net_device->wait_drain);
if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) &&
!net_device->start_remove &&
(hv_ringbuf_avail_percent(&channel->outbound) >
RING_AVAIL_PERCENT_HIWATER || queue_sends < 1))
netif_tx_wake_queue(netdev_get_tx_queue(
ndev, q_idx));
} else {
netdev_err(ndev, "Unknown send completion packet type- "
"%d received!!\n", nvsp_packet->hdr.msg_type);
}
}
/**============================================================================
@brief hdd_tx_fetch_packet_cbk() - Callback function invoked by TL to
fetch a packet for transmission.
@param vosContext : [in] pointer to VOS context
@param staId : [in] Station for which TL is requesting a pkt
@param ac : [in] access category requested by TL
@param pVosPacket : [out] pointer to VOS packet packet pointer
@param pPktMetaInfo : [out] pointer to meta info for the pkt
@return : VOS_STATUS_E_EMPTY if no packets to transmit
: VOS_STATUS_E_FAILURE if any errors encountered
: VOS_STATUS_SUCCESS otherwise
===========================================================================*/
VOS_STATUS hdd_tx_fetch_packet_cbk( v_VOID_t *vosContext,
v_U8_t *pStaId,
WLANTL_ACEnumType ac,
vos_pkt_t **ppVosPacket,
WLANTL_MetaInfoType *pPktMetaInfo )
{
VOS_STATUS status = VOS_STATUS_E_FAILURE;
hdd_adapter_t *pAdapter = NULL;
hdd_context_t *pHddCtx = NULL;
hdd_list_node_t *anchor = NULL;
skb_list_node_t *pktNode = NULL;
struct sk_buff *skb = NULL;
vos_pkt_t *pVosPacket = NULL;
v_MACADDR_t* pDestMacAddress = NULL;
v_TIME_t timestamp;
WLANTL_ACEnumType newAc;
v_SIZE_t size = 0;
tANI_U8 acAdmitted, i;
//Sanity check on inputs
if ( ( NULL == vosContext ) ||
( NULL == pStaId ) ||
( NULL == ppVosPacket ) ||
( NULL == pPktMetaInfo ) )
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR,"%s: Null Params being passed", __FUNCTION__);
return VOS_STATUS_E_FAILURE;
}
//Get the HDD context.
pHddCtx = (hdd_context_t *)vos_get_context( VOS_MODULE_ID_HDD, vosContext );
if(pHddCtx == NULL)
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR,"%s: HDD adapter context is Null", __FUNCTION__);
return VOS_STATUS_E_FAILURE;
}
pAdapter = pHddCtx->sta_to_adapter[*pStaId];
if( NULL == pAdapter )
{
VOS_ASSERT(0);
return VOS_STATUS_E_FAILURE;
}
++pAdapter->hdd_stats.hddTxRxStats.txFetched;
*ppVosPacket = NULL;
//Make sure the AC being asked for is sane
if( ac >= WLANTL_MAX_AC || ac < 0)
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR,"%s: Invalid AC %d passed by TL", __FUNCTION__, ac);
return VOS_STATUS_E_FAILURE;
}
++pAdapter->hdd_stats.hddTxRxStats.txFetchedAC[ac];
#ifdef HDD_WMM_DEBUG
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL,"%s: AC %d passed by TL", __FUNCTION__, ac);
#endif // HDD_WMM_DEBUG
// We find an AC with packets
// or we determine we have no more packets to send
// HDD is not allowed to change AC.
// has this AC been admitted? or
// To allow EAPOL packets when not authenticated
if (unlikely((0==pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcAccessAllowed) &&
(WLAN_HDD_GET_STATION_CTX_PTR(pAdapter))->conn_info.uIsAuthenticated))
{
++pAdapter->hdd_stats.hddTxRxStats.txFetchEmpty;
#ifdef HDD_WMM_DEBUG
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL,
"%s: no packets pending", __FUNCTION__);
#endif // HDD_WMM_DEBUG
return VOS_STATUS_E_FAILURE;
}
// do we have any packets pending in this AC?
hdd_list_size( &pAdapter->wmm_tx_queue[ac], &size );
if( size > 0 )
{
// yes, so process it
#ifdef HDD_WMM_DEBUG
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL,
"%s: AC %d has packets pending", __FUNCTION__, ac);
#endif // HDD_WMM_DEBUG
}
else
{
++pAdapter->hdd_stats.hddTxRxStats.txFetchEmpty;
#ifdef HDD_WMM_DEBUG
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL,
"%s: no packets pending", __FUNCTION__);
#endif // HDD_WMM_DEBUG
return VOS_STATUS_E_FAILURE;
}
//Get the vos packet. I don't want to dequeue and enqueue again if we are out of VOS resources
//This simplifies the locking and unlocking of Tx queue
status = vos_pkt_wrap_data_packet( &pVosPacket,
VOS_PKT_TYPE_TX_802_3_DATA,
NULL, //OS Pkt is not being passed
hdd_tx_low_resource_cbk,
pAdapter );
if (status == VOS_STATUS_E_ALREADY || status == VOS_STATUS_E_RESOURCES)
{
//Remember VOS is in a low resource situation
pAdapter->isVosOutOfResource = VOS_TRUE;
++pAdapter->hdd_stats.hddTxRxStats.txFetchLowResources;
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,"%s: VOSS in Low Resource scenario", __FUNCTION__);
//TL will now think we have no more packets in this AC
return VOS_STATUS_E_FAILURE;
}
//Remove the packet from the queue
spin_lock_bh(&pAdapter->wmm_tx_queue[ac].lock);
status = hdd_list_remove_front( &pAdapter->wmm_tx_queue[ac], &anchor );
spin_unlock_bh(&pAdapter->wmm_tx_queue[ac].lock);
if(VOS_STATUS_SUCCESS == status)
{
//If success then we got a valid packet from some AC
pktNode = list_entry(anchor, skb_list_node_t, anchor);
skb = pktNode->skb;
}
else
{
++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeueError;
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN, "%s: Error in de-queuing "
"skb from Tx queue status = %d", __FUNCTION__, status );
vos_pkt_return_packet(pVosPacket);
return VOS_STATUS_E_FAILURE;
}
//Attach skb to VOS packet.
status = vos_pkt_set_os_packet( pVosPacket, skb );
if (status != VOS_STATUS_SUCCESS)
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,"%s: Error attaching skb", __FUNCTION__);
vos_pkt_return_packet(pVosPacket);
++pAdapter->stats.tx_dropped;
++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeueError;
kfree_skb(skb);
return VOS_STATUS_E_FAILURE;
}
//Just being paranoid. To be removed later
if(pVosPacket == NULL)
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,"%s: VOS packet returned by VOSS is NULL", __FUNCTION__);
++pAdapter->stats.tx_dropped;
++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeueError;
kfree_skb(skb);
return VOS_STATUS_E_FAILURE;
}
//Return VOS packet to TL;
*ppVosPacket = pVosPacket;
//Fill out the meta information needed by TL
//FIXME This timestamp is really the time stamp of wrap_data_packet
vos_pkt_get_timestamp( pVosPacket, ×tamp );
pPktMetaInfo->usTimeStamp = (v_U16_t)timestamp;
if(pAdapter->sessionCtx.station.conn_info.uIsAuthenticated == VOS_TRUE)
pPktMetaInfo->ucIsEapol = 0;
else
pPktMetaInfo->ucIsEapol = hdd_IsEAPOLPacket( pVosPacket ) ? 1 : 0;
#ifdef FEATURE_WLAN_WAPI
// Override usIsEapol value when its zero for WAPI case
pPktMetaInfo->ucIsWai = hdd_IsWAIPacket( pVosPacket ) ? 1 : 0;
#endif /* FEATURE_WLAN_WAPI */
if ((HDD_WMM_USER_MODE_NO_QOS == pHddCtx->cfg_ini->WmmMode) ||
(!pAdapter->hddWmmStatus.wmmQap))
{
// either we don't want QoS or the AP doesn't support QoS
pPktMetaInfo->ucUP = 0;
pPktMetaInfo->ucTID = 0;
}
else
{
/* 1. Check if ACM is set for this AC
* 2. If set, check if this AC had already admitted
* 3. If not already admitted, downgrade the UP to next best UP */
if(!pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcAccessRequired ||
pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcTspecValid)
{
pPktMetaInfo->ucUP = pktNode->userPriority;
pPktMetaInfo->ucTID = pPktMetaInfo->ucUP;
}
else
{
//Downgrade the UP
acAdmitted = pAdapter->hddWmmStatus.wmmAcStatus[ac].wmmAcTspecValid;
newAc = WLANTL_AC_BK;
for (i=ac-1; i>0; i--)
{
if (pAdapter->hddWmmStatus.wmmAcStatus[i].wmmAcAccessRequired == 0)
{
newAc = i;
break;
}
}
pPktMetaInfo->ucUP = hddWmmAcToHighestUp[newAc];
pPktMetaInfo->ucTID = pPktMetaInfo->ucUP;
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_INFO_LOW,"Downgrading UP %d to UP %d ", pktNode->userPriority, pPktMetaInfo->ucUP);
}
}
pPktMetaInfo->ucType = 0; //FIXME Don't know what this is
pPktMetaInfo->ucDisableFrmXtl = 0; //802.3 frame so we need to xlate
if ( 1 < size )
{
pPktMetaInfo->bMorePackets = 1; //HDD has more packets to send
}
else
{
pPktMetaInfo->bMorePackets = 0;
}
//Extract the destination address from ethernet frame
pDestMacAddress = (v_MACADDR_t*)skb->data;
pPktMetaInfo->ucBcast = vos_is_macaddr_broadcast( pDestMacAddress ) ? 1 : 0;
pPktMetaInfo->ucMcast = vos_is_macaddr_group( pDestMacAddress ) ? 1 : 0;
// if we are in a backpressure situation see if we can turn the hose back on
if ( (pAdapter->isTxSuspended[ac]) &&
(size <= HDD_TX_QUEUE_LOW_WATER_MARK) )
{
++pAdapter->hdd_stats.hddTxRxStats.txFetchDePressured;
++pAdapter->hdd_stats.hddTxRxStats.txFetchDePressuredAC[ac];
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_WARN,
"%s: TX queue[%d] re-enabled", __FUNCTION__, ac);
pAdapter->isTxSuspended[ac] = VOS_FALSE;
netif_tx_wake_queue(netdev_get_tx_queue(pAdapter->dev,
skb_get_queue_mapping(skb) ));
}
// We're giving the packet to TL so consider it transmitted from
// a statistics perspective. We account for it here instead of
// when the packet is returned for two reasons. First, TL will
// manipulate the skb to the point where the len field is not
// accurate, leading to inaccurate byte counts if we account for
// it later. Second, TL does not provide any feedback as to
// whether or not the packet was successfully sent over the air,
// so the packet counts will be the same regardless of where we
// account for them
pAdapter->stats.tx_bytes += skb->len;
++pAdapter->stats.tx_packets;
++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeued;
++pAdapter->hdd_stats.hddTxRxStats.txFetchDequeuedAC[ac];
if(pHddCtx->cfg_ini->thermalMitigationEnable)
{
if(mutex_lock_interruptible(&pHddCtx->tmInfo.tmOperationLock))
{
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_ERROR,
"%s: Tm Lock fail", __FUNCTION__);
return VOS_STATUS_E_FAILURE;
}
if(WLAN_HDD_TM_LEVEL_1 < pHddCtx->tmInfo.currentTmLevel)
{
if(0 == pHddCtx->tmInfo.txFrameCount)
{
/* Just recovered from sleep timeout */
pHddCtx->tmInfo.lastOpenTs = timestamp;
}
if(((timestamp - pHddCtx->tmInfo.lastOpenTs) > (pHddCtx->tmInfo.tmAction.txOperationDuration / 10)) &&
(pHddCtx->tmInfo.txFrameCount >= pHddCtx->tmInfo.tmAction.txBlockFrameCountThreshold))
{
spin_lock(&pAdapter->wmm_tx_queue[ac].lock);
/* During TX open duration, TX frame count is larger than threshold
* Block TX during Sleep time */
netif_tx_stop_all_queues(pAdapter->dev);
spin_unlock(&pAdapter->wmm_tx_queue[ac].lock);
pHddCtx->tmInfo.lastblockTs = timestamp;
if(VOS_TIMER_STATE_STOPPED == vos_timer_getCurrentState(&pHddCtx->tmInfo.txSleepTimer))
{
vos_timer_start(&pHddCtx->tmInfo.txSleepTimer, pHddCtx->tmInfo.tmAction.txSleepDuration);
}
}
else if(((timestamp - pHddCtx->tmInfo.lastOpenTs) > (pHddCtx->tmInfo.tmAction.txOperationDuration / 10)) &&
(pHddCtx->tmInfo.txFrameCount < pHddCtx->tmInfo.tmAction.txBlockFrameCountThreshold))
{
/* During TX open duration, TX frame count is less than threshold
* Reset count and timestamp to prepare next cycle */
pHddCtx->tmInfo.lastOpenTs = timestamp;
pHddCtx->tmInfo.txFrameCount = 0;
}
else
{
/* Do Nothing */
}
pHddCtx->tmInfo.txFrameCount++;
}
mutex_unlock(&pHddCtx->tmInfo.tmOperationLock);
}
#ifdef HDD_WMM_DEBUG
VOS_TRACE( VOS_MODULE_ID_HDD, VOS_TRACE_LEVEL_FATAL,"%s: Valid VOS PKT returned to TL", __FUNCTION__);
#endif // HDD_WMM_DEBUG
return status;
}
void xenvif_wake_queue(struct xenvif_queue *queue)
{
struct net_device *dev = queue->vif->dev;
unsigned int id = queue->id;
netif_tx_wake_queue(netdev_get_tx_queue(dev, id));
}
static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = &priv->tx_ring[cq->ring];
struct mlx4_cqe *cqe;
u16 index;
u16 new_index, ring_index;
u32 txbbs_skipped = 0;
u32 cons_index = mcq->cons_index;
int size = cq->size;
u32 size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
if (!priv->port_up)
return;
index = cons_index & size_mask;
cqe = &buf[index];
ring_index = ring->cons & size_mask;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size)) {
/*
* make sure we read the CQE after we read the
* ownership bit
*/
rmb();
/* Skip over last polled CQE */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
txbbs_skipped += ring->last_nr_txbb;
ring_index = (ring_index + ring->last_nr_txbb) & size_mask;
/* free next descriptor */
ring->last_nr_txbb = mlx4_en_free_tx_desc(
priv, ring, ring_index,
!!((ring->cons + txbbs_skipped) &
ring->size));
} while (ring_index != new_index);
++cons_index;
index = cons_index & size_mask;
cqe = &buf[index];
}
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
ring->cons += txbbs_skipped;
/* Wakeup Tx queue if this ring stopped it */
if (unlikely(ring->blocked)) {
if ((u32) (ring->prod - ring->cons) <=
ring->size - HEADROOM - MAX_DESC_TXBBS) {
ring->blocked = 0;
netif_tx_wake_queue(netdev_get_tx_queue(dev, cq->ring));
priv->port_stats.wake_queue++;
}
}
}
