int octeon_report_sent_bytes_to_bql(void *buf, int reqtype) { struct octnet_buf_free_info *finfo; struct sk_buff *skb; struct octeon_soft_command *sc; struct netdev_queue *txq; switch (reqtype) { case REQTYPE_NORESP_NET: case REQTYPE_NORESP_NET_SG: finfo = buf; skb = finfo->skb; break; case REQTYPE_RESP_NET_SG: case REQTYPE_RESP_NET: sc = buf; skb = sc->callback_arg; break; default: return 0; } txq = netdev_get_tx_queue(skb->dev, skb_get_queue_mapping(skb)); netdev_tx_sent_queue(txq, skb->len); return netif_xmit_stopped(txq); }
static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb) { struct mlx5_wq_cyc *wq = &sq->wq; u16 pi = sq->pc & wq->sz_m1; struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; struct mlx5_wqe_eth_seg *eseg = &wqe->eth; struct mlx5_wqe_data_seg *dseg; u8 opcode = MLX5_OPCODE_SEND; dma_addr_t dma_addr = 0; bool bf = false; u16 headlen; u16 ds_cnt; u16 ihs; int i; memset(wqe, 0, sizeof(*wqe)); if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM; else sq->stats.csum_offload_none++; if (sq->cc != sq->prev_cc) { sq->prev_cc = sq->cc; sq->bf_budget = (sq->cc == sq->pc) ? MLX5E_SQ_BF_BUDGET : 0; } if (skb_is_gso(skb)) { u32 payload_len; eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size); opcode = MLX5_OPCODE_LSO; ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); payload_len = skb->len - ihs; MLX5E_TX_SKB_CB(skb)->num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs; sq->stats.tso_packets++; sq->stats.tso_bytes += payload_len; } else { bf = sq->bf_budget && !skb->xmit_more && !skb_shinfo(skb)->nr_frags; ihs = mlx5e_get_inline_hdr_size(sq, skb, bf); MLX5E_TX_SKB_CB(skb)->num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN); } skb_copy_from_linear_data(skb, eseg->inline_hdr_start, ihs); skb_pull_inline(skb, ihs); eseg->inline_hdr_sz = cpu_to_be16(ihs); ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start), MLX5_SEND_WQE_DS); dseg = (struct mlx5_wqe_data_seg *)cseg + ds_cnt; MLX5E_TX_SKB_CB(skb)->num_dma = 0; headlen = skb_headlen(skb); if (headlen) { dma_addr = dma_map_single(sq->pdev, skb->data, headlen, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) goto dma_unmap_wqe_err; dseg->addr = cpu_to_be64(dma_addr); dseg->lkey = sq->mkey_be; dseg->byte_count = cpu_to_be32(headlen); mlx5e_dma_push(sq, dma_addr, headlen); MLX5E_TX_SKB_CB(skb)->num_dma++; dseg++; } for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; int fsz = skb_frag_size(frag); dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) goto dma_unmap_wqe_err; dseg->addr = cpu_to_be64(dma_addr); dseg->lkey = sq->mkey_be; dseg->byte_count = cpu_to_be32(fsz); mlx5e_dma_push(sq, dma_addr, fsz); MLX5E_TX_SKB_CB(skb)->num_dma++; dseg++; } ds_cnt += MLX5E_TX_SKB_CB(skb)->num_dma; cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); sq->skb[pi] = skb; MLX5E_TX_SKB_CB(skb)->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); sq->pc += MLX5E_TX_SKB_CB(skb)->num_wqebbs; netdev_tx_sent_queue(sq->txq, MLX5E_TX_SKB_CB(skb)->num_bytes); if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM))) { netif_tx_stop_queue(sq->txq); sq->stats.stopped++; } if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) { int bf_sz = 0; if (bf && sq->uar_bf_map) bf_sz = MLX5E_TX_SKB_CB(skb)->num_wqebbs << 3; cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; mlx5e_tx_notify_hw(sq, wqe, bf_sz); } /* fill sq edge with nops to avoid wqe wrap around */ while ((sq->pc & wq->sz_m1) > sq->edge) mlx5e_send_nop(sq, false); sq->bf_budget = bf ? sq->bf_budget - 1 : 0; sq->stats.packets++; return NETDEV_TX_OK; dma_unmap_wqe_err: sq->stats.dropped++; mlx5e_dma_unmap_wqe_err(sq, skb); dev_kfree_skb_any(skb); return NETDEV_TX_OK; }
/** * nfp_net_tx() - Main transmit entry point * @skb: SKB to transmit * @netdev: netdev structure * * Return: NETDEV_TX_OK on success. */ static int nfp_net_tx(struct sk_buff *skb, struct net_device *netdev) { struct nfp_net *nn = netdev_priv(netdev); const struct skb_frag_struct *frag; struct nfp_net_r_vector *r_vec; struct nfp_net_tx_desc *txd, txdg; struct nfp_net_tx_buf *txbuf; struct nfp_net_tx_ring *tx_ring; struct netdev_queue *nd_q; dma_addr_t dma_addr; unsigned int fsize; int f, nr_frags; int wr_idx; u16 qidx; qidx = skb_get_queue_mapping(skb); tx_ring = &nn->tx_rings[qidx]; r_vec = tx_ring->r_vec; nd_q = netdev_get_tx_queue(nn->netdev, qidx); nr_frags = skb_shinfo(skb)->nr_frags; if (unlikely(nfp_net_tx_full(tx_ring, nr_frags + 1))) { nn_warn_ratelimit(nn, "TX ring %d busy. wrp=%u rdp=%u\n", qidx, tx_ring->wr_p, tx_ring->rd_p); netif_tx_stop_queue(nd_q); u64_stats_update_begin(&r_vec->tx_sync); r_vec->tx_busy++; u64_stats_update_end(&r_vec->tx_sync); return NETDEV_TX_BUSY; } /* Start with the head skbuf */ dma_addr = dma_map_single(&nn->pdev->dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); if (dma_mapping_error(&nn->pdev->dev, dma_addr)) goto err_free; wr_idx = tx_ring->wr_p % tx_ring->cnt; /* Stash the soft descriptor of the head then initialize it */ txbuf = &tx_ring->txbufs[wr_idx]; txbuf->skb = skb; txbuf->dma_addr = dma_addr; txbuf->fidx = -1; txbuf->pkt_cnt = 1; txbuf->real_len = skb->len; /* Build TX descriptor */ txd = &tx_ring->txds[wr_idx]; txd->offset_eop = (nr_frags == 0) ? PCIE_DESC_TX_EOP : 0; txd->dma_len = cpu_to_le16(skb_headlen(skb)); nfp_desc_set_dma_addr(txd, dma_addr); txd->data_len = cpu_to_le16(skb->len); txd->flags = 0; txd->mss = 0; txd->l4_offset = 0; nfp_net_tx_tso(nn, r_vec, txbuf, txd, skb); nfp_net_tx_csum(nn, r_vec, txbuf, txd, skb); if (skb_vlan_tag_present(skb) && nn->ctrl & NFP_NET_CFG_CTRL_TXVLAN) { txd->flags |= PCIE_DESC_TX_VLAN; txd->vlan = cpu_to_le16(skb_vlan_tag_get(skb)); } /* Gather DMA */ if (nr_frags > 0) { /* all descs must match except for in addr, length and eop */ txdg = *txd; for (f = 0; f < nr_frags; f++) { frag = &skb_shinfo(skb)->frags[f]; fsize = skb_frag_size(frag); dma_addr = skb_frag_dma_map(&nn->pdev->dev, frag, 0, fsize, DMA_TO_DEVICE); if (dma_mapping_error(&nn->pdev->dev, dma_addr)) goto err_unmap; wr_idx = (wr_idx + 1) % tx_ring->cnt; tx_ring->txbufs[wr_idx].skb = skb; tx_ring->txbufs[wr_idx].dma_addr = dma_addr; tx_ring->txbufs[wr_idx].fidx = f; txd = &tx_ring->txds[wr_idx]; *txd = txdg; txd->dma_len = cpu_to_le16(fsize); nfp_desc_set_dma_addr(txd, dma_addr); txd->offset_eop = (f == nr_frags - 1) ? PCIE_DESC_TX_EOP : 0; } u64_stats_update_begin(&r_vec->tx_sync); r_vec->tx_gather++; u64_stats_update_end(&r_vec->tx_sync); } netdev_tx_sent_queue(nd_q, txbuf->real_len); tx_ring->wr_p += nr_frags + 1; if (nfp_net_tx_ring_should_stop(tx_ring)) nfp_net_tx_ring_stop(nd_q, tx_ring); tx_ring->wr_ptr_add += nr_frags + 1; if (!skb->xmit_more || netif_xmit_stopped(nd_q)) { /* force memory write before we let HW know */ wmb(); nfp_qcp_wr_ptr_add(tx_ring->qcp_q, tx_ring->wr_ptr_add); tx_ring->wr_ptr_add = 0; } skb_tx_timestamp(skb); return NETDEV_TX_OK; err_unmap: --f; while (f >= 0) { frag = &skb_shinfo(skb)->frags[f]; dma_unmap_page(&nn->pdev->dev, tx_ring->txbufs[wr_idx].dma_addr, skb_frag_size(frag), DMA_TO_DEVICE); tx_ring->txbufs[wr_idx].skb = NULL; tx_ring->txbufs[wr_idx].dma_addr = 0; tx_ring->txbufs[wr_idx].fidx = -2; wr_idx = wr_idx - 1; if (wr_idx < 0) wr_idx += tx_ring->cnt; } dma_unmap_single(&nn->pdev->dev, tx_ring->txbufs[wr_idx].dma_addr, skb_headlen(skb), DMA_TO_DEVICE); tx_ring->txbufs[wr_idx].skb = NULL; tx_ring->txbufs[wr_idx].dma_addr = 0; tx_ring->txbufs[wr_idx].fidx = -2; err_free: nn_warn_ratelimit(nn, "Failed to map DMA TX buffer\n"); u64_stats_update_begin(&r_vec->tx_sync); r_vec->tx_errors++; u64_stats_update_end(&r_vec->tx_sync); dev_kfree_skb_any(skb); return NETDEV_TX_OK; }
static inline int dma_xmit(struct sk_buff *skb, struct net_device *dev, END_DEVICE *ei_local, int gmac_no) { struct netdev_queue *txq; dma_addr_t frag_addr; u32 frag_size, nr_desc; u32 txd_info3, txd_info4; #if defined (CONFIG_RAETH_SG_DMA_TX) u32 i, nr_frags; const skb_frag_t *tx_frag; const struct skb_shared_info *shinfo; #else #define nr_frags 0 #endif #if defined (CONFIG_RA_HW_NAT) || defined (CONFIG_RA_HW_NAT_MODULE) if (ra_sw_nat_hook_tx != NULL) { #if defined (CONFIG_RA_HW_NAT_WIFI) || defined (CONFIG_RA_HW_NAT_PCI) if (IS_DPORT_PPE_VALID(skb)) gmac_no = PSE_PORT_PPE; else #endif if (ra_sw_nat_hook_tx(skb, gmac_no) == 0) { dev_kfree_skb(skb); return NETDEV_TX_OK; } } #endif txd_info3 = TX3_QDMA_SWC; if (gmac_no != PSE_PORT_PPE) { u32 QID = M2Q_table[(skb->mark & 0x3f)]; if (QID < 8 && M2Q_wan_lan) { #if defined (CONFIG_PSEUDO_SUPPORT) if (gmac_no == PSE_PORT_GMAC2) QID += 8; #elif defined (CONFIG_RAETH_HW_VLAN_TX) if ((skb_vlan_tag_get(skb) & VLAN_VID_MASK) > 1) QID += 8; #endif } txd_info3 |= TX3_QDMA_QID(QID); } txd_info4 = TX4_DMA_FPORT(gmac_no); #if defined (CONFIG_RAETH_CHECKSUM_OFFLOAD) if (skb->ip_summed == CHECKSUM_PARTIAL) txd_info4 |= TX4_DMA_TUI_CO(7); #endif #if defined (CONFIG_RAETH_HW_VLAN_TX) if (skb_vlan_tag_present(skb)) txd_info4 |= (0x10000 | skb_vlan_tag_get(skb)); #endif #if defined (CONFIG_RAETH_SG_DMA_TX) shinfo = skb_shinfo(skb); #endif #if defined (CONFIG_RAETH_TSO) /* fill MSS info in tcp checksum field */ if (shinfo->gso_size) { u32 hdr_len; if (!(shinfo->gso_type & (SKB_GSO_TCPV4|SKB_GSO_TCPV6))) { dev_kfree_skb(skb); return NETDEV_TX_OK; } if (skb_header_cloned(skb)) { if (pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) { dev_kfree_skb(skb); return NETDEV_TX_OK; } } hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb)); if (hdr_len >= skb->len) { dev_kfree_skb(skb); return NETDEV_TX_OK; } tcp_hdr(skb)->check = htons(shinfo->gso_size); txd_info4 |= TX4_DMA_TSO; } #endif nr_desc = DIV_ROUND_UP(skb_headlen(skb), TXD_MAX_SEG_SIZE); #if defined (CONFIG_RAETH_SG_DMA_TX) nr_frags = (u32)shinfo->nr_frags; for (i = 0; i < nr_frags; i++) { tx_frag = &shinfo->frags[i]; nr_desc += DIV_ROUND_UP(skb_frag_size(tx_frag), TXD_MAX_SEG_SIZE); } #endif txq = netdev_get_tx_queue(dev, 0); /* flush main skb part before spin_lock() */ frag_size = (u32)skb_headlen(skb); frag_addr = dma_map_single(NULL, skb->data, frag_size, DMA_TO_DEVICE); /* protect TX ring access (from eth2/eth3 queues) */ spin_lock(&ei_local->page_lock); /* check nr_desc+2 free descriptors (2 need to prevent head/tail overlap) */ if (ei_local->txd_pool_free_num < (nr_desc+2)) { spin_unlock(&ei_local->page_lock); netif_tx_stop_queue(txq); #if defined (CONFIG_RAETH_DEBUG) if (net_ratelimit()) printk("%s: QDMA TX pool is run out! (GMAC: %d)\n", RAETH_DEV_NAME, gmac_no); #endif return NETDEV_TX_BUSY; } qdma_write_skb_fragment(ei_local, frag_addr, frag_size, txd_info3, txd_info4, skb, nr_frags == 0); #if defined (CONFIG_RAETH_SG_DMA_TX) for (i = 0; i < nr_frags; i++) { tx_frag = &shinfo->frags[i]; frag_size = skb_frag_size(tx_frag); frag_addr = skb_frag_dma_map(NULL, tx_frag, 0, frag_size, DMA_TO_DEVICE); qdma_write_skb_fragment(ei_local, frag_addr, frag_size, txd_info3, txd_info4, skb, i == nr_frags - 1); } #endif #if defined (CONFIG_RAETH_BQL) netdev_tx_sent_queue(txq, skb->len); #endif #if !defined (CONFIG_RAETH_BQL) || !defined (CONFIG_SMP) /* smp_mb() already inlined in netdev_tx_sent_queue */ wmb(); #endif /* kick the QDMA TX */ sysRegWrite(QTX_CTX_PTR, (u32)get_txd_ptr_phy(ei_local, ei_local->txd_last_idx)); spin_unlock(&ei_local->page_lock); return NETDEV_TX_OK; }
static int xlgmac_xmit(struct sk_buff *skb, struct net_device *netdev) { struct xlgmac_pdata *pdata = netdev_priv(netdev); struct xlgmac_pkt_info *tx_pkt_info; struct xlgmac_desc_ops *desc_ops; struct xlgmac_channel *channel; struct xlgmac_hw_ops *hw_ops; struct netdev_queue *txq; struct xlgmac_ring *ring; int ret; desc_ops = &pdata->desc_ops; hw_ops = &pdata->hw_ops; XLGMAC_PR("skb->len = %d\n", skb->len); channel = pdata->channel_head + skb->queue_mapping; txq = netdev_get_tx_queue(netdev, channel->queue_index); ring = channel->tx_ring; tx_pkt_info = &ring->pkt_info; if (skb->len == 0) { netif_err(pdata, tx_err, netdev, "empty skb received from stack\n"); dev_kfree_skb_any(skb); return NETDEV_TX_OK; } /* Prepare preliminary packet info for TX */ memset(tx_pkt_info, 0, sizeof(*tx_pkt_info)); xlgmac_prep_tx_pkt(pdata, ring, skb, tx_pkt_info); /* Check that there are enough descriptors available */ ret = xlgmac_maybe_stop_tx_queue(channel, ring, tx_pkt_info->desc_count); if (ret) return ret; ret = xlgmac_prep_tso(skb, tx_pkt_info); if (ret) { netif_err(pdata, tx_err, netdev, "error processing TSO packet\n"); dev_kfree_skb_any(skb); return ret; } xlgmac_prep_vlan(skb, tx_pkt_info); if (!desc_ops->map_tx_skb(channel, skb)) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; } /* Report on the actual number of bytes (to be) sent */ netdev_tx_sent_queue(txq, tx_pkt_info->tx_bytes); /* Configure required descriptor fields for transmission */ hw_ops->dev_xmit(channel); if (netif_msg_pktdata(pdata)) xlgmac_print_pkt(netdev, skb, true); /* Stop the queue in advance if there may not be enough descriptors */ xlgmac_maybe_stop_tx_queue(channel, ring, XLGMAC_TX_MAX_DESC_NR); return NETDEV_TX_OK; }
netdev_tx_t mpodp_start_xmit(struct sk_buff *skb, struct net_device *netdev) { struct mpodp_if_priv *priv = netdev_priv(netdev); struct mpodp_tx *tx; struct dma_async_tx_descriptor *dma_txd; struct mpodp_cache_entry *entry; int ret; uint8_t fifo_mode; int16_t requested_engine; struct mpodp_pkt_hdr *hdr; uint32_t tx_autoloop_next; uint32_t tx_submitted, tx_next, tx_done; uint32_t tx_mppa_idx; int qidx; unsigned long flags = 0; struct mpodp_txq *txq; /* Fetch HW queue selected by the kernel */ qidx = skb_get_queue_mapping(skb); txq = &priv->txqs[qidx]; if (atomic_read(&priv->reset) == 1) { mpodp_clean_tx_unlocked(priv, txq, -1); goto addr_error; } tx_submitted = atomic_read(&txq->submitted); /* Compute txd id */ tx_next = (tx_submitted + 1); if (tx_next == txq->size) tx_next = 0; /* MPPA H2C Entry to use */ tx_mppa_idx = atomic_read(&txq->autoloop_cur); tx_done = atomic_read(&txq->done); if (tx_done != tx_submitted && ((txq->ring[tx_done].jiffies + msecs_to_jiffies(5) >= jiffies) || (tx_submitted < tx_done && tx_submitted + txq->size - tx_done >= TX_POLL_THRESHOLD) || (tx_submitted >= tx_done && tx_submitted - tx_done >= TX_POLL_THRESHOLD))) { mpodp_clean_tx_unlocked(priv, txq, -1); } /* Check if there are txd available */ if (tx_next == atomic_read(&txq->done)) { /* Ring is full */ if (netif_msg_tx_err(priv)) netdev_err(netdev, "txq[%d]: ring full \n", txq->id); netif_tx_stop_queue(txq->txq); return NETDEV_TX_BUSY; } tx = &(txq->ring[tx_submitted]); entry = &(txq->cache[tx_mppa_idx]); /* take the time */ mppa_pcie_time_get(priv->tx_time, &tx->time); /* configure channel */ tx->dst_addr = entry->addr; /* Check the provided address */ ret = mppa_pcie_dma_check_addr(priv->pdata, tx->dst_addr, &fifo_mode, &requested_engine); if (ret) { if (netif_msg_tx_err(priv)) netdev_err(netdev, "txq[%d] tx[%d]: invalid send address %llx\n", txq->id, tx_submitted, tx->dst_addr); goto addr_error; } if (!fifo_mode) { if (netif_msg_tx_err(priv)) netdev_err(netdev, "txq[%d] tx[%d]: %llx is not a PCI2Noc addres\n", txq->id, tx_submitted, tx->dst_addr); goto addr_error; } if (requested_engine >= MPODP_NOC_CHAN_COUNT) { if (netif_msg_tx_err(priv)) netdev_err(netdev, "txq[%d] tx[%d]: address %llx using NoC engine out of range (%d >= %d)\n", txq->id, tx_submitted, tx->dst_addr, requested_engine, MPODP_NOC_CHAN_COUNT); goto addr_error; } tx->chanidx = requested_engine; /* The packet needs a header to determine size,timestamp, etc. * Add it */ if (skb_headroom(skb) < sizeof(struct mpodp_pkt_hdr)) { struct sk_buff *skb_new; skb_new = skb_realloc_headroom(skb, sizeof(struct mpodp_pkt_hdr)); if (!skb_new) { netdev->stats.tx_errors++; kfree_skb(skb); return NETDEV_TX_OK; } kfree_skb(skb); skb = skb_new; } hdr = (struct mpodp_pkt_hdr *) skb_push(skb, sizeof(struct mpodp_pkt_hdr)); hdr->timestamp = priv->packet_id; hdr->info._.pkt_id = priv->packet_id; hdr->info.dword = 0ULL; hdr->info._.pkt_size = skb->len; /* Also count the header size */ hdr->info._.pkt_id = priv->packet_id; priv->packet_id++; /* save skb to free it later */ tx->skb = skb; tx->len = skb->len; /* prepare sg */ if (map_skb(&priv->pdev->dev, skb, tx)){ if (netif_msg_tx_err(priv)) netdev_err(netdev, "tx %d: failed to map skb to dma\n", tx_submitted); goto busy; } if (priv->n_txqs > MPODP_NOC_CHAN_COUNT) spin_lock_irqsave(&priv->tx_lock[requested_engine], flags); /* Prepare slave args */ priv->tx_config[requested_engine].cfg.dst_addr = tx->dst_addr; priv->tx_config[requested_engine].requested_engine = requested_engine; /* FIFO mode, direction, latency were filled at setup */ if (dmaengine_slave_config(priv->tx_chan[requested_engine], &priv->tx_config[requested_engine].cfg)) { /* board has reset, wait for reset of netdev */ netif_tx_stop_queue(txq->txq); netif_carrier_off(netdev); if (netif_msg_tx_err(priv)) netdev_err(netdev, "txq[%d] tx[%d]: cannot configure channel\n", txq->id, tx_submitted); goto busy; } /* get transfer descriptor */ dma_txd = dmaengine_prep_slave_sg(priv->tx_chan[requested_engine], tx->sg, tx->sg_len, DMA_MEM_TO_DEV, 0); if (dma_txd == NULL) { /* dmaengine_prep_slave_sg failed, retry */ if (netif_msg_tx_err(priv)) netdev_err(netdev, "txq[%d] tx[%d]: cannot get dma descriptor\n", txq->id, tx_submitted); goto busy; } if (netif_msg_tx_queued(priv)) netdev_info(netdev, "txq[%d] tx[%d]: transfer start (submitted: %d done: %d) len=%d, sg_len=%d\n", txq->id, tx_submitted, tx_next, atomic_read(&txq->done), tx->len, tx->sg_len); skb_orphan(skb); /* submit and issue descriptor */ tx->jiffies = jiffies; tx->cookie = dmaengine_submit(dma_txd); dma_async_issue_pending(priv->tx_chan[requested_engine]); if (priv->n_txqs > MPODP_NOC_CHAN_COUNT) spin_unlock_irqrestore(&priv->tx_lock[requested_engine], flags); /* Count number of bytes on the fly for DQL */ netdev_tx_sent_queue(txq->txq, skb->len); if (test_bit(__QUEUE_STATE_STACK_XOFF, &txq->txq->state)){ /* We reached over the limit of DQL. Try to clean some * tx so we are rescheduled right now */ mpodp_clean_tx_unlocked(priv, txq, -1); } /* Increment tail pointer locally */ atomic_set(&txq->submitted, tx_next); /* Update H2C entry offset */ tx_autoloop_next = tx_mppa_idx + 1; if (tx_autoloop_next == txq->cached_head) tx_autoloop_next = 0; atomic_set(&txq->autoloop_cur, tx_autoloop_next); skb_tx_timestamp(skb); /* Check if there is room for another txd * or stop the queue if there is not */ tx_next = (tx_next + 1); if (tx_next == txq->size) tx_next = 0; if (tx_next == atomic_read(&txq->done)) { if (netif_msg_tx_queued(priv)) netdev_info(netdev, "txq[%d]: ring full \n", txq->id); netif_tx_stop_queue(txq->txq); } return NETDEV_TX_OK; busy: unmap_skb(&priv->pdev->dev, skb, tx); return NETDEV_TX_BUSY; addr_error: netdev->stats.tx_dropped++; dev_kfree_skb(skb); /* We can't do anything, just stop the queue artificially */ netif_tx_stop_queue(txq->txq); return NETDEV_TX_OK; }
static inline int dma_xmit(struct sk_buff* skb, struct net_device *dev, END_DEVICE *ei_local, int gmac_no) { struct netdev_queue *txq; dma_addr_t frag_addr; u32 frag_size, nr_desc; u32 next_idx, desc_odd = 0; u32 txd_info2 = 0, txd_info4; #if defined (CONFIG_RAETH_SG_DMA_TX) u32 i, nr_frags; const skb_frag_t *tx_frag; const struct skb_shared_info *shinfo; #else #define nr_frags 0 #endif #if defined (CONFIG_RA_HW_NAT) || defined (CONFIG_RA_HW_NAT_MODULE) if (ra_sw_nat_hook_tx != NULL) { #if defined (CONFIG_RA_HW_NAT_WIFI) || defined (CONFIG_RA_HW_NAT_PCI) if (IS_DPORT_PPE_VALID(skb)) gmac_no = PSE_PORT_PPE; else #endif if (ra_sw_nat_hook_tx(skb, gmac_no) == 0) { dev_kfree_skb(skb); return NETDEV_TX_OK; } } #endif #if !defined (RAETH_HW_PADPKT) if (skb->len < ei_local->min_pkt_len) { if (skb_padto(skb, ei_local->min_pkt_len)) { #if defined (CONFIG_RAETH_DEBUG) if (net_ratelimit()) printk(KERN_ERR "%s: skb_padto failed\n", RAETH_DEV_NAME); #endif return NETDEV_TX_OK; } skb_put(skb, ei_local->min_pkt_len - skb->len); } #endif #if defined (CONFIG_RALINK_MT7620) if (gmac_no == PSE_PORT_PPE) txd_info4 = TX4_DMA_FP_BMAP(0x80); /* P7 */ else #if defined (CONFIG_RAETH_HAS_PORT5) && !defined (CONFIG_RAETH_HAS_PORT4) && !defined (CONFIG_RAETH_ESW) txd_info4 = TX4_DMA_FP_BMAP(0x20); /* P5 */ #elif defined (CONFIG_RAETH_HAS_PORT4) && !defined (CONFIG_RAETH_HAS_PORT5) && !defined (CONFIG_RAETH_ESW) txd_info4 = TX4_DMA_FP_BMAP(0x10); /* P4 */ #else txd_info4 = 0; /* routing by DA */ #endif #elif defined (CONFIG_RALINK_MT7621) txd_info4 = TX4_DMA_FPORT(gmac_no); #else txd_info4 = (TX4_DMA_QN(3) | TX4_DMA_PN(gmac_no)); #endif #if defined (CONFIG_RAETH_CHECKSUM_OFFLOAD) && !defined (RAETH_SDMA) if (skb->ip_summed == CHECKSUM_PARTIAL) txd_info4 |= TX4_DMA_TUI_CO(7); #endif #if defined (CONFIG_RAETH_HW_VLAN_TX) if (skb_vlan_tag_present(skb)) { #if defined (RAETH_HW_VLAN4K) txd_info4 |= (0x10000 | skb_vlan_tag_get(skb)); #else u32 vlan_tci = skb_vlan_tag_get(skb); txd_info4 |= (TX4_DMA_INSV | TX4_DMA_VPRI(vlan_tci)); txd_info4 |= (u32)ei_local->vlan_4k_map[(vlan_tci & VLAN_VID_MASK)]; #endif } #endif #if defined (CONFIG_RAETH_SG_DMA_TX) shinfo = skb_shinfo(skb); #endif #if defined (CONFIG_RAETH_TSO) /* fill MSS info in tcp checksum field */ if (shinfo->gso_size) { u32 hdr_len; if (!(shinfo->gso_type & (SKB_GSO_TCPV4|SKB_GSO_TCPV6))) { dev_kfree_skb(skb); return NETDEV_TX_OK; } if (skb_header_cloned(skb)) { if (pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) { dev_kfree_skb(skb); return NETDEV_TX_OK; } } hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb)); if (hdr_len >= skb->len) { dev_kfree_skb(skb); return NETDEV_TX_OK; } tcp_hdr(skb)->check = htons(shinfo->gso_size); txd_info4 |= TX4_DMA_TSO; } #endif nr_desc = DIV_ROUND_UP(skb_headlen(skb), TXD_MAX_SEG_SIZE); #if defined (CONFIG_RAETH_SG_DMA_TX) nr_frags = (u32)shinfo->nr_frags; for (i = 0; i < nr_frags; i++) { tx_frag = &shinfo->frags[i]; nr_desc += DIV_ROUND_UP(skb_frag_size(tx_frag), TXD_MAX_SEG_SIZE); } #endif nr_desc = DIV_ROUND_UP(nr_desc, 2); txq = netdev_get_tx_queue(dev, 0); /* flush main skb part before spin_lock() */ frag_size = (u32)skb_headlen(skb); frag_addr = dma_map_single(NULL, skb->data, frag_size, DMA_TO_DEVICE); /* protect TX ring access (from eth2/eth3 queues) */ spin_lock(&ei_local->page_lock); /* check nr_desc+1 free descriptors */ next_idx = (ei_local->txd_last_idx + nr_desc) % NUM_TX_DESC; if (ei_local->txd_buff[ei_local->txd_last_idx] || ei_local->txd_buff[next_idx]) { spin_unlock(&ei_local->page_lock); netif_tx_stop_queue(txq); #if defined (CONFIG_RAETH_DEBUG) if (net_ratelimit()) printk("%s: PDMA TX ring is full! (GMAC: %d)\n", RAETH_DEV_NAME, gmac_no); #endif return NETDEV_TX_BUSY; } pdma_write_skb_fragment(ei_local, frag_addr, frag_size, &desc_odd, &txd_info2, txd_info4, skb, nr_frags == 0); #if defined (CONFIG_RAETH_SG_DMA_TX) for (i = 0; i < nr_frags; i++) { tx_frag = &shinfo->frags[i]; frag_size = skb_frag_size(tx_frag); frag_addr = skb_frag_dma_map(NULL, tx_frag, 0, frag_size, DMA_TO_DEVICE); pdma_write_skb_fragment(ei_local, frag_addr, frag_size, &desc_odd, &txd_info2, txd_info4, skb, i == nr_frags - 1); } #endif #if defined (CONFIG_RAETH_BQL) netdev_tx_sent_queue(txq, skb->len); #endif #if !defined (CONFIG_RAETH_BQL) || !defined (CONFIG_SMP) /* smp_mb() already inlined in netdev_tx_sent_queue */ wmb(); #endif /* kick the DMA TX */ sysRegWrite(TX_CTX_IDX0, cpu_to_le32(ei_local->txd_last_idx)); spin_unlock(&ei_local->page_lock); return NETDEV_TX_OK; }
static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb) { struct mlx5_wq_cyc *wq = &sq->wq; u16 pi = sq->pc & wq->sz_m1; struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi); struct mlx5e_tx_wqe_info *wi = &sq->wqe_info[pi]; struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; struct mlx5_wqe_eth_seg *eseg = &wqe->eth; struct mlx5_wqe_data_seg *dseg; unsigned char *skb_data = skb->data; unsigned int skb_len = skb->len; u8 opcode = MLX5_OPCODE_SEND; dma_addr_t dma_addr = 0; unsigned int num_bytes; bool bf = false; u16 headlen; u16 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; u16 ihs; int i; memset(wqe, 0, sizeof(*wqe)); if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM; if (skb->encapsulation) { eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM | MLX5_ETH_WQE_L4_INNER_CSUM; sq->stats.csum_partial_inner++; } else { eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM; sq->stats.csum_partial++; } } else sq->stats.csum_none++; if (sq->cc != sq->prev_cc) { sq->prev_cc = sq->cc; sq->bf_budget = (sq->cc == sq->pc) ? MLX5E_SQ_BF_BUDGET : 0; } if (skb_is_gso(skb)) { eseg->mss = cpu_to_be16(skb_shinfo(skb)->gso_size); opcode = MLX5_OPCODE_LSO; if (skb->encapsulation) { ihs = skb_inner_transport_header(skb) - skb->data + inner_tcp_hdrlen(skb); sq->stats.tso_inner_packets++; sq->stats.tso_inner_bytes += skb->len - ihs; } else { ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); sq->stats.tso_packets++; sq->stats.tso_bytes += skb->len - ihs; } num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs; } else { bf = sq->bf_budget && !skb->xmit_more && !skb_shinfo(skb)->nr_frags; ihs = mlx5e_get_inline_hdr_size(sq, skb, bf); num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN); } wi->num_bytes = num_bytes; if (skb_vlan_tag_present(skb)) { mlx5e_insert_vlan(eseg->inline_hdr_start, skb, ihs, &skb_data, &skb_len); ihs += VLAN_HLEN; } else { memcpy(eseg->inline_hdr_start, skb_data, ihs); mlx5e_tx_skb_pull_inline(&skb_data, &skb_len, ihs); } eseg->inline_hdr_sz = cpu_to_be16(ihs); ds_cnt += DIV_ROUND_UP(ihs - sizeof(eseg->inline_hdr_start), MLX5_SEND_WQE_DS); dseg = (struct mlx5_wqe_data_seg *)cseg + ds_cnt; wi->num_dma = 0; headlen = skb_len - skb->data_len; if (headlen) { dma_addr = dma_map_single(sq->pdev, skb_data, headlen, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) goto dma_unmap_wqe_err; dseg->addr = cpu_to_be64(dma_addr); dseg->lkey = sq->mkey_be; dseg->byte_count = cpu_to_be32(headlen); mlx5e_dma_push(sq, dma_addr, headlen, MLX5E_DMA_MAP_SINGLE); wi->num_dma++; dseg++; } for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; int fsz = skb_frag_size(frag); dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) goto dma_unmap_wqe_err; dseg->addr = cpu_to_be64(dma_addr); dseg->lkey = sq->mkey_be; dseg->byte_count = cpu_to_be32(fsz); mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE); wi->num_dma++; dseg++; } ds_cnt += wi->num_dma; cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); sq->skb[pi] = skb; wi->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); sq->pc += wi->num_wqebbs; if (unlikely(MLX5E_TX_HW_STAMP(sq->channel->priv, skb))) skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; netdev_tx_sent_queue(sq->txq, wi->num_bytes); if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_SQ_STOP_ROOM))) { netif_tx_stop_queue(sq->txq); sq->stats.queue_stopped++; } if (!skb->xmit_more || netif_xmit_stopped(sq->txq)) { int bf_sz = 0; if (bf && sq->uar_bf_map) bf_sz = wi->num_wqebbs << 3; cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; mlx5e_tx_notify_hw(sq, &wqe->ctrl, bf_sz); } sq->bf_budget = bf ? sq->bf_budget - 1 : 0; /* fill sq edge with nops to avoid wqe wrap around */ while ((sq->pc & wq->sz_m1) > sq->edge) mlx5e_send_nop(sq, false); sq->stats.packets++; sq->stats.bytes += num_bytes; return NETDEV_TX_OK; dma_unmap_wqe_err: sq->stats.queue_dropped++; mlx5e_dma_unmap_wqe_err(sq, wi->num_dma); dev_kfree_skb_any(skb); return NETDEV_TX_OK; }