static void mlx5e_close_sq(struct mlx5e_sq *sq) { clear_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state); napi_synchronize(&sq->channel->napi); /* prevent netif_tx_wake_queue */ netif_tx_disable_queue(sq->txq); /* ensure hw is notified of all pending wqes */ if (mlx5e_sq_has_room_for(sq, 1)) mlx5e_send_nop(sq); mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR); while (sq->cc != sq->pc) /* wait till sq is empty */ msleep(20); /* avoid destroying sq before mlx5e_poll_tx_cq() is done with it */ napi_synchronize(&sq->channel->napi); mlx5e_disable_sq(sq); mlx5e_destroy_sq(sq); }
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq) { struct mlx5e_sq *sq; u32 dma_fifo_cc; u32 nbytes; u16 npkts; u16 sqcc; int i; /* avoid accessing cq (dma coherent memory) if not needed */ if (!test_and_clear_bit(MLX5E_CQ_HAS_CQES, &cq->flags)) return false; sq = container_of(cq, struct mlx5e_sq, cq); npkts = 0; nbytes = 0; /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), * otherwise a cq overrun may occur */ sqcc = sq->cc; /* avoid dirtying sq cache line every cqe */ dma_fifo_cc = sq->dma_fifo_cc; for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { struct mlx5_cqe64 *cqe; u16 wqe_counter; bool last_wqe; cqe = mlx5e_get_cqe(cq); if (!cqe) break; mlx5_cqwq_pop(&cq->wq); wqe_counter = be16_to_cpu(cqe->wqe_counter); do { struct sk_buff *skb; u16 ci; int j; last_wqe = (sqcc == wqe_counter); ci = sqcc & sq->wq.sz_m1; skb = sq->skb[ci]; if (unlikely(!skb)) { /* nop */ sq->stats.nop++; sqcc++; continue; } for (j = 0; j < MLX5E_TX_SKB_CB(skb)->num_dma; j++) { dma_addr_t addr; u32 size; mlx5e_dma_get(sq, dma_fifo_cc, &addr, &size); dma_fifo_cc++; dma_unmap_single(sq->pdev, addr, size, DMA_TO_DEVICE); } npkts++; nbytes += MLX5E_TX_SKB_CB(skb)->num_bytes; sqcc += MLX5E_TX_SKB_CB(skb)->num_wqebbs; dev_kfree_skb(skb); } while (!last_wqe); } mlx5_cqwq_update_db_record(&cq->wq); /* ensure cq space is freed before enabling more cqes */ wmb(); sq->dma_fifo_cc = dma_fifo_cc; sq->cc = sqcc; netdev_tx_completed_queue(sq->txq, npkts, nbytes); if (netif_tx_queue_stopped(sq->txq) && mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) && likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) { netif_tx_wake_queue(sq->txq); sq->stats.wake++; } if (i == MLX5E_TX_CQ_POLL_BUDGET) { set_bit(MLX5E_CQ_HAS_CQES, &cq->flags); return true; } return false; }
static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb) { struct mlx5_wq_cyc *wq = &sq->wq; u16 pi = sq->pc & wq->sz_m1; struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; struct mlx5_wqe_eth_seg *eseg = &wqe->eth; struct mlx5_wqe_data_seg *dseg; u8 opcode = MLX5_OPCODE_SEND; dma_addr_t dma_addr = 0; bool bf = false; u16 headlen; u16 ds_cnt; u16 ihs; int i; memset(wqe, 0, sizeof(*wqe)); if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; else sq->stats.csum_offload_none++; if (sq->cc != sq->prev_cc) { sq->prev_cc = sq->cc; sq->bf_budget = (sq->cc == sq->pc) ? MLX5E_SQ_BF_BUDGET : 0; } if (skb_is_gso(skb)) { u32 payload_len; eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size); opcode = MLX5_OPCODE_LSO; ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); payload_len = skb->len - ihs; MLX5E_TX_SKB_CB(skb)->num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs; sq->stats.tso_packets++; sq->stats.tso_bytes += payload_len; } else { bf = sq->bf_budget && !skb->xmit_more && !skb_shinfo(skb)->nr_frags; ihs = mlx5e_get_inline_hdr_size(sq, skb, bf); MLX5E_TX_SKB_CB(skb)->num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN); } skb_copy_from_linear_data(skb, eseg->inline_hdr_start, ihs); skb_pull_inline(skb, ihs); eseg->inline_hdr_sz = cpu_to_be16(ihs); ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start), MLX5_SEND_WQE_DS); dseg = (struct mlx5_wqe_data_seg *)cseg + ds_cnt; MLX5E_TX_SKB_CB(skb)->num_dma = 0; headlen = skb_headlen(skb); if (headlen) { dma_addr = dma_map_single(sq->pdev, skb->data, headlen, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) goto dma_unmap_wqe_err; dseg->addr = cpu_to_be64(dma_addr); dseg->lkey = sq->mkey_be; dseg->byte_count = cpu_to_be32(headlen); mlx5e_dma_push(sq, dma_addr, headlen); MLX5E_TX_SKB_CB(skb)->num_dma++; dseg++; } for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; int fsz = skb_frag_size(frag); dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) goto dma_unmap_wqe_err; dseg->addr = cpu_to_be64(dma_addr); dseg->lkey = sq->mkey_be; dseg->byte_count = cpu_to_be32(fsz); mlx5e_dma_push(sq, dma_addr, fsz); MLX5E_TX_SKB_CB(skb)->num_dma++; dseg++; } ds_cnt += MLX5E_TX_SKB_CB(skb)->num_dma; cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); sq->skb[pi] = skb; MLX5E_TX_SKB_CB(skb)->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); sq->pc += MLX5E_TX_SKB_CB(skb)->num_wqebbs; netdev_tx_sent_queue(sq->txq, MLX5E_TX_SKB_CB(skb)->num_bytes); if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM))) { netif_tx_stop_queue(sq->txq); sq->stats.stopped++; } if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) { int bf_sz = 0; if (bf && sq->uar_bf_map) bf_sz = MLX5E_TX_SKB_CB(skb)->num_wqebbs << 3; cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; mlx5e_tx_notify_hw(sq, wqe, bf_sz); } /* fill sq edge with nops to avoid wqe wrap around */ while ((sq->pc & wq->sz_m1) > sq->edge) mlx5e_send_nop(sq, false); sq->bf_budget = bf ? sq->bf_budget - 1 : 0; sq->stats.packets++; return NETDEV_TX_OK; dma_unmap_wqe_err: sq->stats.dropped++; mlx5e_dma_unmap_wqe_err(sq, skb); dev_kfree_skb_any(skb); return NETDEV_TX_OK; }
bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq) { struct mlx5_cqe64 *cqe; struct mlx5e_sq *sq; u32 dma_fifo_cc; u32 nbytes; u16 npkts; u16 sqcc; int i; sq = container_of(cq, struct mlx5e_sq, cq); if (unlikely(test_bit(MLX5E_SQ_TX_TIMEOUT, &sq->state))) return false; npkts = 0; nbytes = 0; /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), * otherwise a cq overrun may occur */ sqcc = sq->cc; /* avoid dirtying sq cache line every cqe */ dma_fifo_cc = sq->dma_fifo_cc; cqe = mlx5e_get_cqe(cq); for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) { u16 wqe_counter; bool last_wqe; if (!cqe) break; mlx5_cqwq_pop(&cq->wq); mlx5e_prefetch_cqe(cq); wqe_counter = be16_to_cpu(cqe->wqe_counter); do { struct mlx5e_tx_wqe_info *wi; struct sk_buff *skb; u16 ci; int j; last_wqe = (sqcc == wqe_counter); ci = sqcc & sq->wq.sz_m1; skb = sq->skb[ci]; wi = &sq->wqe_info[ci]; if (unlikely(!skb)) { /* nop */ sqcc++; continue; } if (unlikely(MLX5E_TX_HW_STAMP(sq->channel->priv, skb))) { struct skb_shared_hwtstamps hwts; mlx5e_fill_hwstamp(&sq->cq.channel->priv->tstamp, &hwts, get_cqe_ts(cqe)); skb_tstamp_tx(skb, &hwts); } for (j = 0; j < wi->num_dma; j++) { struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, dma_fifo_cc++); mlx5e_tx_dma_unmap(sq->pdev, dma); } npkts++; nbytes += wi->num_bytes; sqcc += wi->num_wqebbs; dev_kfree_skb(skb); } while (!last_wqe); cqe = mlx5e_get_cqe(cq); } mlx5_cqwq_update_db_record(&cq->wq); /* ensure cq space is freed before enabling more cqes */ wmb(); sq->dma_fifo_cc = dma_fifo_cc; sq->cc = sqcc; netdev_tx_completed_queue(sq->txq, npkts, nbytes); if (netif_tx_queue_stopped(sq->txq) && mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM) && likely(test_bit(MLX5E_SQ_STATE_WAKE_TXQ_ENABLE, &sq->state))) { netif_tx_wake_queue(sq->txq); sq->stats.queue_wake++; } return (i == MLX5E_TX_CQ_POLL_BUDGET); }
static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb) { struct mlx5_wq_cyc *wq = &sq->wq; u16 pi = sq->pc & wq->sz_m1; struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); struct mlx5e_tx_wqe_info *wi = &sq->wqe_info[pi]; struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; struct mlx5_wqe_eth_seg *eseg = &wqe->eth; struct mlx5_wqe_data_seg *dseg; unsigned char *skb_data = skb->data; unsigned int skb_len = skb->len; u8 opcode = MLX5_OPCODE_SEND; dma_addr_t dma_addr = 0; unsigned int num_bytes; bool bf = false; u16 headlen; u16 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; u16 ihs; int i; memset(wqe, 0, sizeof(*wqe)); if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM; if (skb->encapsulation) { eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM | MLX5_ETH_WQE_L4_INNER_CSUM; sq->stats.csum_partial_inner++; } else { eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM; sq->stats.csum_partial++; } } else sq->stats.csum_none++; if (sq->cc != sq->prev_cc) { sq->prev_cc = sq->cc; sq->bf_budget = (sq->cc == sq->pc) ? MLX5E_SQ_BF_BUDGET : 0; } if (skb_is_gso(skb)) { eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size); opcode = MLX5_OPCODE_LSO; if (skb->encapsulation) { ihs = skb_inner_transport_header(skb) - skb->data + inner_tcp_hdrlen(skb); sq->stats.tso_inner_packets++; sq->stats.tso_inner_bytes += skb->len - ihs; } else { ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); sq->stats.tso_packets++; sq->stats.tso_bytes += skb->len - ihs; } num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs; } else { bf = sq->bf_budget && !skb->xmit_more && !skb_shinfo(skb)->nr_frags; ihs = mlx5e_get_inline_hdr_size(sq, skb, bf); num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN); } wi->num_bytes = num_bytes; if (skb_vlan_tag_present(skb)) { mlx5e_insert_vlan(eseg->inline_hdr_start, skb, ihs, &skb_data, &skb_len); ihs += VLAN_HLEN; } else { memcpy(eseg->inline_hdr_start, skb_data, ihs); mlx5e_tx_skb_pull_inline(&skb_data, &skb_len, ihs); } eseg->inline_hdr_sz = cpu_to_be16(ihs); ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start), MLX5_SEND_WQE_DS); dseg = (struct mlx5_wqe_data_seg *)cseg + ds_cnt; wi->num_dma = 0; headlen = skb_len - skb->data_len; if (headlen) { dma_addr = dma_map_single(sq->pdev, skb_data, headlen, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) goto dma_unmap_wqe_err; dseg->addr = cpu_to_be64(dma_addr); dseg->lkey = sq->mkey_be; dseg->byte_count = cpu_to_be32(headlen); mlx5e_dma_push(sq, dma_addr, headlen, MLX5E_DMA_MAP_SINGLE); wi->num_dma++; dseg++; } for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; int fsz = skb_frag_size(frag); dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) goto dma_unmap_wqe_err; dseg->addr = cpu_to_be64(dma_addr); dseg->lkey = sq->mkey_be; dseg->byte_count = cpu_to_be32(fsz); mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE); wi->num_dma++; dseg++; } ds_cnt += wi->num_dma; cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); sq->skb[pi] = skb; wi->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); sq->pc += wi->num_wqebbs; if (unlikely(MLX5E_TX_HW_STAMP(sq->channel->priv, skb))) skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; netdev_tx_sent_queue(sq->txq, wi->num_bytes); if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM))) { netif_tx_stop_queue(sq->txq); sq->stats.queue_stopped++; } if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) { int bf_sz = 0; if (bf && sq->uar_bf_map) bf_sz = wi->num_wqebbs << 3; cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; mlx5e_tx_notify_hw(sq, &wqe->ctrl, bf_sz); } sq->bf_budget = bf ? sq->bf_budget - 1 : 0; /* fill sq edge with nops to avoid wqe wrap around */ while ((sq->pc & wq->sz_m1) > sq->edge) mlx5e_send_nop(sq, false); sq->stats.packets++; sq->stats.bytes += num_bytes; return NETDEV_TX_OK; dma_unmap_wqe_err: sq->stats.queue_dropped++; mlx5e_dma_unmap_wqe_err(sq, wi->num_dma); dev_kfree_skb_any(skb); return NETDEV_TX_OK; }