struct mlx5_cqe64 * mlx5e_get_cqe(struct mlx5e_cq *cq) { struct mlx5_cqe64 *cqe; cqe = mlx5_cqwq_get_wqe(&cq->wq, mlx5_cqwq_get_ci(&cq->wq)); if ((cqe->op_own ^ mlx5_cqwq_get_wrap_cnt(&cq->wq)) & MLX5_CQE_OWNER_MASK) return (NULL); mlx5_cqwq_pop(&cq->wq); /* ensure cqe content is read after cqe ownership bit */ rmb(); return (cqe); }
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq) { struct mlx5e_sq *sq; u32 dma_fifo_cc; u32 nbytes; u16 npkts; u16 sqcc; int i; /* avoid accessing cq (dma coherent memory) if not needed */ if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags)) return false; sq = container_of(cq, struct mlx5e_sq, cq); npkts = 0; nbytes = 0; /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), * otherwise a cq overrun may occur */ sqcc = sq->cc; /* avoid dirtying sq cache line every cqe */ dma_fifo_cc = sq->dma_fifo_cc; for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { struct mlx5_cqe64 *cqe; u16 wqe_counter; bool last_wqe; cqe = mlx5e_get_cqe(cq); if (!cqe) break; mlx5_cqwq_pop(&cq->wq); wqe_counter = be16_to_cpu(cqe->wqe_counter); do { struct sk_buff *skb; u16 ci; int j; last_wqe = (sqcc == wqe_counter); ci = sqcc & sq->wq.sz_m1; skb = sq->skb[ci]; if (unlikely(!skb)) { /* nop */ sq->stats.nop++; sqcc++; continue; } for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) { dma_addr_t addr; u32 size; mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size); dma_fifo_cc++; dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE); } npkts++; nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes; sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs; dev_kfree_skb(skb); } while (!last_wqe); } mlx5_cqwq_update_db_record(&cq->wq); /* ensure cq space is freed before enabling more cqes */ wmb(); sq->dma_fifo_cc = dma_fifo_cc; sq->cc = sqcc; netdev_tx_completed_queue(sq->txq, npkts, nbytes); if (netif_tx_queue_stopped(sq->txq) && mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) && likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) { netif_tx_wake_queue(sq->txq); sq->stats.wake++; } if (i == MLX5E_TX_CQ_POLL_BUDGET) { set_bit(MLX5E_CQ_HAS_CQES, &cq->flags); return true; } return false; }
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq) { struct mlx5_cqe64 *cqe; struct mlx5e_sq *sq; u32 dma_fifo_cc; u32 nbytes; u16 npkts; u16 sqcc; int i; sq = container_of(cq, struct mlx5e_sq, cq); if (unlikely(test_bit(MLX5E_SQ_TX_TIMEOUT, &sq->state))) return false; npkts = 0; nbytes = 0; /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), * otherwise a cq overrun may occur */ sqcc = sq->cc; /* avoid dirtying sq cache line every cqe */ dma_fifo_cc = sq->dma_fifo_cc; cqe = mlx5e_get_cqe(cq); for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { u16 wqe_counter; bool last_wqe; if (!cqe) break; mlx5_cqwq_pop(&cq->wq); mlx5e_prefetch_cqe(cq); wqe_counter = be16_to_cpu(cqe->wqe_counter); do { struct mlx5e_tx_wqe_info *wi; struct sk_buff *skb; u16 ci; int j; last_wqe = (sqcc == wqe_counter); ci = sqcc & sq->wq.sz_m1; skb = sq->skb[ci]; wi = &sq->wqe_info[ci]; if (unlikely(!skb)) { /* nop */ sqcc++; continue; } if (unlikely(MLX5E_TX_HW_STAMP(sq->channel->priv, skb))) { struct skb_shared_hwtstamps hwts; mlx5e_fill_hwstamp(&sq->cq.channel->priv->tstamp, &hwts, get_cqe_ts(cqe)); skb_tstamp_tx(skb, &hwts); } for (j = 0; j < wi->num_dma; j++) { struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, dma_fifo_cc++); mlx5e_tx_dma_unmap(sq->pdev, dma); } npkts++; nbytes += wi->num_bytes; sqcc += wi->num_wqebbs; dev_kfree_skb(skb); } while (!last_wqe); cqe = mlx5e_get_cqe(cq); } mlx5_cqwq_update_db_record(&cq->wq); /* ensure cq space is freed before enabling more cqes */ wmb(); sq->dma_fifo_cc = dma_fifo_cc; sq->cc = sqcc; netdev_tx_completed_queue(sq->txq, npkts, nbytes); if (netif_tx_queue_stopped(sq->txq) && mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) && likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) { netif_tx_wake_queue(sq->txq); sq->stats.queue_wake++; } return (i == MLX5E_TX_CQ_POLL_BUDGET); }