void mlx4_en_rx_que(void *context, int pending)
{
struct mlx4_en_cq *cq;
cq = context;
while (mlx4_en_poll_rx_cq(cq, MLX4_EN_MAX_RX_POLL)
== MLX4_EN_MAX_RX_POLL);
mlx4_en_arm_cq(cq->dev->if_softc, cq);
}
void mlx4_en_rx_irq(struct mlx4_cq *mcq)
{
struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
struct mlx4_en_priv *priv = netdev_priv(cq->dev);
if (priv->port_up)
napi_schedule(&cq->napi);
else
mlx4_en_arm_cq(priv, cq);
}
void mlx4_en_rx_irq(struct mlx4_cq *mcq)
{
struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
struct mlx4_en_priv *priv = netdev_priv(cq->dev);
int done;
done = mlx4_en_poll_rx_cq(cq, MLX4_EN_MAX_RX_POLL);
if (done == MLX4_EN_MAX_RX_POLL)
taskqueue_enqueue(cq->tq, &cq->cq_task);
else
mlx4_en_arm_cq(priv, cq);
}
/* Rx CQ polling - called by NAPI */
int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
{
struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
struct net_device *dev = cq->dev;
struct mlx4_en_priv *priv = netdev_priv(dev);
int done;
done = mlx4_en_process_rx_cq(dev, cq, budget);
/* If we used up all the quota - we're probably not done yet... */
if (done == budget)
INC_PERF_COUNTER(priv->pstats.napi_quota);
else {
/* Done for now */
napi_complete(napi);
mlx4_en_arm_cq(priv, cq);
}
return done;
}
/* Rx CQ polling - called by NAPI */
int mlx4_en_poll_rx_cq(struct net_device *poll_dev, int *budget)
{
struct mlx4_en_cq *cq = poll_dev->priv;
struct net_device *dev = cq->dev;
struct mlx4_en_priv *priv = netdev_priv(dev);
int done;
int work = min(*budget, poll_dev->quota);
done = mlx4_en_process_rx_cq(dev, cq, work);
dev->quota -= done;
*budget -= done;
/* If we used up all the quota - we're probably not done yet... */
if (done == work) {
INC_PERF_COUNTER(priv->pstats.napi_quota);
return 1;
}
/* Done for now */
netif_rx_complete(poll_dev);
mlx4_en_arm_cq(priv, cq);
return 0;
}
static void mlx4_en_create_rl_res(struct mlx4_en_priv *priv,
int ring_id, u8 rate_index)
{
struct mlx4_en_cq *cq;
struct mlx4_en_tx_ring *tx_ring;
struct mlx4_en_dev *mdev = priv->mdev;
int err = 0;
int node = 0;
int j;
if (priv->tx_ring[ring_id]) {
/* Ring already exists, needs activation */
/* Make sure drbr queue has no left overs from before */
tx_ring = priv->tx_ring[ring_id];
goto activate;
}
err = mlx4_en_create_cq(priv, &priv->tx_cq[ring_id],
MLX4_EN_DEF_RL_TX_RING_SIZE, ring_id, TX, node);
if (err) {
en_err(priv, "Failed to create rate limit tx CQ, ring index %u, rate %u\n", ring_id, rate_index);
goto err_create_cq;
}
err = mlx4_en_create_tx_ring(priv, &priv->tx_ring[ring_id],
MLX4_EN_DEF_RL_TX_RING_SIZE, TXBB_SIZE, node, ring_id);
if (err) {
en_err(priv, "Failed to create rate limited tx ring %u, rate %u\n", ring_id, rate_index);
goto err_create_ring;
}
tx_ring = priv->tx_ring[ring_id];
activate:
sysctl_ctx_init(&tx_ring->rl_data.rl_stats_ctx);
tx_ring->rl_data.rate_index = rate_index;
/* Default moderation */
cq = priv->tx_cq[ring_id];
cq->moder_cnt = priv->tx_frames;
cq->moder_time = priv->tx_usecs;
mutex_lock(&mdev->state_lock);
if (!priv->port_up) {
/* No need activating resources, start_port will take care of that */
tx_ring->rl_data.user_valid = true;
mutex_unlock(&mdev->state_lock);
return;
}
/* Activate resources */
err = mlx4_en_activate_cq(priv, cq, ring_id);
if (err) {
en_err(priv, "Failed activating Rate Limit Tx CQ\n");
goto err_activate_resources;
}
err = mlx4_en_set_cq_moder(priv, cq);
if (err) {
en_err(priv, "Failed setting cq moderation parameters");
mlx4_en_deactivate_cq(priv, cq);
goto err_activate_resources;
}
en_dbg(DRV, priv, "Resetting index of CQ:%d to -1\n", ring_id);
cq->buf->wqe_index = cpu_to_be16(0xffff);
err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn,
MLX4_EN_DEF_RL_USER_PRIO);
if (err) {
en_err(priv, "Failed activating rate limit TX ring\n");
mlx4_en_deactivate_cq(priv, cq);
goto err_activate_resources;
}
/* Arm CQ for TX completions */
mlx4_en_arm_cq(priv, cq);
/* Set initial ownership of all Tx TXBBs to SW (1) */
for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE)
*((u32 *) (tx_ring->buf + j)) = INIT_OWNER_BIT;
/* Set ring as valid */
tx_ring->rl_data.user_valid = true;
mutex_unlock(&mdev->state_lock);
priv->rate_limit_tx_ring_num++;
/* Add rate limit statistics to sysctl if debug option was enabled */
if (show_rl_sysctl_info)
mlx4_en_rate_limit_sysctl_stat(priv, ring_id);
return;
err_activate_resources:
mlx4_en_invalidate_rl_ring(priv, ring_id);
mlx4_en_rl_reused_index_insert(priv, ring_id);
atomic_subtract_int(&priv->rate_limits[rate_index].ref, 1);
mutex_unlock(&mdev->state_lock);
return;
err_create_ring:
if (priv->tx_cq[ring_id])
mlx4_en_destroy_cq(priv, &priv->tx_cq[ring_id]);
err_create_cq:
mlx4_en_rl_reused_index_insert(priv, ring_id);
atomic_subtract_int(&priv->rate_limits[rate_index].ref, 1);
}
u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring,
int index, u64 timestamp,
int napi_mode)
{
struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
struct mlx4_en_rx_alloc frame = {
.page = tx_info->page,
.dma = tx_info->map0_dma,
};
if (!mlx4_en_rx_recycle(ring->recycle_ring, &frame)) {
dma_unmap_page(priv->ddev, tx_info->map0_dma,
PAGE_SIZE, priv->dma_dir);
put_page(tx_info->page);
}
return tx_info->nr_txbb;
}
int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
int cnt = 0;
/* Skip last polled descriptor */
ring->cons += ring->last_nr_txbb;
en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
ring->cons, ring->prod);
if ((u32) (ring->prod - ring->cons) > ring->size) {
if (netif_msg_tx_err(priv))
en_warn(priv, "Tx consumer passed producer!\n");
return 0;
}
while (ring->cons != ring->prod) {
ring->last_nr_txbb = ring->free_tx_desc(priv, ring,
ring->cons & ring->size_mask,
0, 0 /* Non-NAPI caller */);
ring->cons += ring->last_nr_txbb;
cnt++;
}
if (ring->tx_queue)
netdev_tx_reset_queue(ring->tx_queue);
if (cnt)
en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt);
return cnt;
}
bool mlx4_en_process_tx_cq(struct net_device *dev,
struct mlx4_en_cq *cq, int napi_budget)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->type][cq->ring];
struct mlx4_cqe *cqe;
u16 index, ring_index, stamp_index;
u32 txbbs_skipped = 0;
u32 txbbs_stamp = 0;
u32 cons_index = mcq->cons_index;
int size = cq->size;
u32 size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
u32 packets = 0;
u32 bytes = 0;
int factor = priv->cqe_factor;
int done = 0;
int budget = priv->tx_work_limit;
u32 last_nr_txbb;
u32 ring_cons;
if (unlikely(!priv->port_up))
return true;
netdev_txq_bql_complete_prefetchw(ring->tx_queue);
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
last_nr_txbb = READ_ONCE(ring->last_nr_txbb);
ring_cons = READ_ONCE(ring->cons);
ring_index = ring_cons & size_mask;
stamp_index = ring_index;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size) && (done < budget)) {
u16 new_index;
/*
* make sure we read the CQE after we read the
* ownership bit
*/
dma_rmb();
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe;
en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
cqe_err->vendor_err_syndrome,
cqe_err->syndrome);
}
/* Skip over last polled CQE */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
u64 timestamp = 0;
txbbs_skipped += last_nr_txbb;
ring_index = (ring_index + last_nr_txbb) & size_mask;
if (unlikely(ring->tx_info[ring_index].ts_requested))
timestamp = mlx4_en_get_cqe_ts(cqe);
/* free next descriptor */
last_nr_txbb = ring->free_tx_desc(
priv, ring, ring_index,
timestamp, napi_budget);
mlx4_en_stamp_wqe(priv, ring, stamp_index,
!!((ring_cons + txbbs_stamp) &
ring->size));
stamp_index = ring_index;
txbbs_stamp = txbbs_skipped;
packets++;
bytes += ring->tx_info[ring_index].nr_bytes;
} while ((++done < budget) && (ring_index != new_index));
++cons_index;
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
}
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
/* we want to dirty this cache line once */
WRITE_ONCE(ring->last_nr_txbb, last_nr_txbb);
WRITE_ONCE(ring->cons, ring_cons + txbbs_skipped);
if (cq->type == TX_XDP)
return done < budget;
netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
/* Wakeup Tx queue if this stopped, and ring is not full.
*/
if (netif_tx_queue_stopped(ring->tx_queue) &&
!mlx4_en_is_tx_ring_full(ring)) {
netif_tx_wake_queue(ring->tx_queue);
ring->wake_queue++;
}
return done < budget;
}
void mlx4_en_tx_irq(struct mlx4_cq *mcq)
{
struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
struct mlx4_en_priv *priv = netdev_priv(cq->dev);
if (likely(priv->port_up))
napi_schedule_irqoff(&cq->napi);
else
mlx4_en_arm_cq(priv, cq);
}
/* TX CQ polling - called by NAPI */
int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget)
{
struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
struct net_device *dev = cq->dev;
struct mlx4_en_priv *priv = netdev_priv(dev);
bool clean_complete;
clean_complete = mlx4_en_process_tx_cq(dev, cq, budget);
if (!clean_complete)
return budget;
napi_complete(napi);
mlx4_en_arm_cq(priv, cq);
return 0;
}
static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring,
u32 index,
unsigned int desc_size)
{
u32 copy = (ring->size - index) << LOG_TXBB_SIZE;
int i;
for (i = desc_size - copy - 4; i >= 0; i -= 4) {
if ((i & (TXBB_SIZE - 1)) == 0)
wmb();
*((u32 *) (ring->buf + i)) =
*((u32 *) (ring->bounce_buf + copy + i));
}
for (i = copy - 4; i >= 4 ; i -= 4) {
if ((i & (TXBB_SIZE - 1)) == 0)
wmb();
*((u32 *)(ring->buf + (index << LOG_TXBB_SIZE) + i)) =
*((u32 *) (ring->bounce_buf + i));
}
/* Return real descriptor location */
return ring->buf + (index << LOG_TXBB_SIZE);
}
