static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn,
struct mlx5_fpga_dma_buf *buf)
{
struct mlx5_wqe_data_seg *data;
unsigned int ix;
int err = 0;
err = mlx5_fpga_conn_map_buf(conn, buf);
if (unlikely(err))
goto out;
if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) {
mlx5_fpga_conn_unmap_buf(conn, buf);
return -EBUSY;
}
ix = conn->qp.rq.pc & (conn->qp.rq.size - 1);
data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix);
data->byte_count = cpu_to_be32(buf->sg[0].size);
data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key);
data->addr = cpu_to_be64(buf->sg[0].dma_addr);
conn->qp.rq.pc++;
conn->qp.rq.bufs[ix] = buf;
/* Make sure that descriptors are written before doorbell record. */
dma_wmb();
*conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff);
out:
return err;
}
static void bgmac_dma_rx_update_index(struct bgmac *bgmac,
struct bgmac_dma_ring *ring)
{
dma_wmb();
bgmac_write(bgmac, ring->mmio_base + BGMAC_DMA_RX_INDEX,
ring->index_base +
ring->end * sizeof(struct bgmac_dma_desc));
}
static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe)
{
/* ensure wqe is visible to device before updating doorbell record */
dma_wmb();
*conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc);
/* Make sure that doorbell record is visible before ringing */
wmb();
mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET);
}
static netdev_tx_t xge_start_xmit(struct sk_buff *skb, struct net_device *ndev)
{
struct xge_pdata *pdata = netdev_priv(ndev);
struct device *dev = &pdata->pdev->dev;
struct xge_desc_ring *tx_ring;
struct xge_raw_desc *raw_desc;
static dma_addr_t dma_addr;
u64 addr_lo, addr_hi;
void *pkt_buf;
u8 tail;
u16 len;
tx_ring = pdata->tx_ring;
tail = tx_ring->tail;
len = skb_headlen(skb);
raw_desc = &tx_ring->raw_desc[tail];
if (!is_tx_slot_available(raw_desc)) {
netif_stop_queue(ndev);
return NETDEV_TX_BUSY;
}
/* Packet buffers should be 64B aligned */
pkt_buf = dma_zalloc_coherent(dev, XGENE_ENET_STD_MTU, &dma_addr,
GFP_ATOMIC);
if (unlikely(!pkt_buf)) {
dev_kfree_skb_any(skb);
return NETDEV_TX_OK;
}
memcpy(pkt_buf, skb->data, len);
addr_hi = GET_BITS(NEXT_DESC_ADDRH, le64_to_cpu(raw_desc->m1));
addr_lo = GET_BITS(NEXT_DESC_ADDRL, le64_to_cpu(raw_desc->m1));
raw_desc->m1 = cpu_to_le64(SET_BITS(NEXT_DESC_ADDRL, addr_lo) |
SET_BITS(NEXT_DESC_ADDRH, addr_hi) |
SET_BITS(PKT_ADDRH,
upper_32_bits(dma_addr)));
tx_ring->pkt_info[tail].skb = skb;
tx_ring->pkt_info[tail].dma_addr = dma_addr;
tx_ring->pkt_info[tail].pkt_buf = pkt_buf;
dma_wmb();
raw_desc->m0 = cpu_to_le64(SET_BITS(PKT_ADDRL, dma_addr) |
SET_BITS(PKT_SIZE, len) |
SET_BITS(E, 0));
skb_tx_timestamp(skb);
xge_wr_csr(pdata, DMATXCTRL, 1);
tx_ring->tail = (tail + 1) & (XGENE_ENET_NUM_DESC - 1);
return NETDEV_TX_OK;
}
static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
{
struct bm_mc *mc = &portal->mc;
union bm_mc_result *rr = mc->rr + mc->rridx;
DPAA_ASSERT(mc->state == mc_user);
dma_wmb();
mc->cr->_ncw_verb = myverb | mc->vbit;
dpaa_flush(mc->cr);
dpaa_invalidate_touch_ro(rr);
#ifdef CONFIG_FSL_DPAA_CHECKING
mc->state = mc_hw;
#endif
}
static void mlx4_en_tx_write_desc(struct mlx4_en_tx_ring *ring,
struct mlx4_en_tx_desc *tx_desc,
union mlx4_wqe_qpn_vlan qpn_vlan,
int desc_size, int bf_index,
__be32 op_own, bool bf_ok,
bool send_doorbell)
{
tx_desc->ctrl.qpn_vlan = qpn_vlan;
if (bf_ok) {
op_own |= htonl((bf_index & 0xffff) << 8);
/* Ensure new descriptor hits memory
* before setting ownership of this descriptor to HW
*/
dma_wmb();
tx_desc->ctrl.owner_opcode = op_own;
wmb();
mlx4_bf_copy(ring->bf.reg + ring->bf.offset, &tx_desc->ctrl,
desc_size);
wmb();
ring->bf.offset ^= ring->bf.buf_size;
} else {
/* Ensure new descriptor hits memory
* before setting ownership of this descriptor to HW
*/
dma_wmb();
tx_desc->ctrl.owner_opcode = op_own;
if (send_doorbell)
mlx4_en_xmit_doorbell(ring);
else
ring->xmit_more++;
}
}
static int xge_refill_buffers(struct net_device *ndev, u32 nbuf)
{
struct xge_pdata *pdata = netdev_priv(ndev);
struct xge_desc_ring *ring = pdata->rx_ring;
const u8 slots = XGENE_ENET_NUM_DESC - 1;
struct device *dev = &pdata->pdev->dev;
struct xge_raw_desc *raw_desc;
u64 addr_lo, addr_hi;
u8 tail = ring->tail;
struct sk_buff *skb;
dma_addr_t dma_addr;
u16 len;
int i;
for (i = 0; i < nbuf; i++) {
raw_desc = &ring->raw_desc[tail];
len = XGENE_ENET_STD_MTU;
skb = netdev_alloc_skb(ndev, len);
if (unlikely(!skb))
return -ENOMEM;
dma_addr = dma_map_single(dev, skb->data, len, DMA_FROM_DEVICE);
if (dma_mapping_error(dev, dma_addr)) {
netdev_err(ndev, "DMA mapping error\n");
dev_kfree_skb_any(skb);
return -EINVAL;
}
ring->pkt_info[tail].skb = skb;
ring->pkt_info[tail].dma_addr = dma_addr;
addr_hi = GET_BITS(NEXT_DESC_ADDRH, le64_to_cpu(raw_desc->m1));
addr_lo = GET_BITS(NEXT_DESC_ADDRL, le64_to_cpu(raw_desc->m1));
raw_desc->m1 = cpu_to_le64(SET_BITS(NEXT_DESC_ADDRL, addr_lo) |
SET_BITS(NEXT_DESC_ADDRH, addr_hi) |
SET_BITS(PKT_ADDRH,
upper_32_bits(dma_addr)));
dma_wmb();
raw_desc->m0 = cpu_to_le64(SET_BITS(PKT_ADDRL, dma_addr) |
SET_BITS(E, 1));
tail = (tail + 1) & slots;
}
ring->tail = tail;
return 0;
}
static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc,
const struct sk_buff *skb,
const struct skb_shared_info *shinfo,
void *fragptr)
{
struct mlx4_wqe_inline_seg *inl = &tx_desc->inl;
int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof(*inl);
unsigned int hlen = skb_headlen(skb);
if (skb->len <= spc) {
if (likely(skb->len >= MIN_PKT_LEN)) {
inl->byte_count = cpu_to_be32(1 << 31 | skb->len);
} else {
inl->byte_count = cpu_to_be32(1 << 31 | MIN_PKT_LEN);
memset(((void *)(inl + 1)) + skb->len, 0,
MIN_PKT_LEN - skb->len);
}
skb_copy_from_linear_data(skb, inl + 1, hlen);
if (shinfo->nr_frags)
memcpy(((void *)(inl + 1)) + hlen, fragptr,
skb_frag_size(&shinfo->frags[0]));
} else {
inl->byte_count = cpu_to_be32(1 << 31 | spc);
if (hlen <= spc) {
skb_copy_from_linear_data(skb, inl + 1, hlen);
if (hlen < spc) {
memcpy(((void *)(inl + 1)) + hlen,
fragptr, spc - hlen);
fragptr += spc - hlen;
}
inl = (void *) (inl + 1) + spc;
memcpy(((void *)(inl + 1)), fragptr, skb->len - spc);
} else {
skb_copy_from_linear_data(skb, inl + 1, spc);
inl = (void *) (inl + 1) + spc;
skb_copy_from_linear_data_offset(skb, spc, inl + 1,
hlen - spc);
if (shinfo->nr_frags)
memcpy(((void *)(inl + 1)) + hlen - spc,
fragptr,
skb_frag_size(&shinfo->frags[0]));
}
dma_wmb();
inl->byte_count = cpu_to_be32(1 << 31 | (skb->len - spc));
}
}
static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
{
struct bm_rcr *rcr = &portal->rcr;
struct bm_rcr_entry *rcursor;
DPAA_ASSERT(rcr->busy);
DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
DPAA_ASSERT(rcr->available >= 1);
dma_wmb();
rcursor = rcr->cursor;
rcursor->_ncw_verb = myverb | rcr->vbit;
dpaa_flush(rcursor);
rcr_inc(rcr);
rcr->available--;
#ifdef CONFIG_FSL_DPAA_CHECKING
rcr->busy = 0;
#endif
}
static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb)
{
struct qtnf_pcie_pearl_state *ps = get_bus_priv(bus);
struct qtnf_pcie_bus_priv *priv = &ps->base;
dma_addr_t txbd_paddr, skb_paddr;
struct qtnf_pearl_tx_bd *txbd;
unsigned long flags;
int len, i;
u32 info;
int ret = 0;
spin_lock_irqsave(&priv->tx_lock, flags);
if (!qtnf_tx_queue_ready(ps)) {
if (skb->dev) {
netif_tx_stop_all_queues(skb->dev);
priv->tx_stopped = 1;
}
spin_unlock_irqrestore(&priv->tx_lock, flags);
return NETDEV_TX_BUSY;
}
i = priv->tx_bd_w_index;
priv->tx_skb[i] = skb;
len = skb->len;
skb_paddr = pci_map_single(priv->pdev, skb->data,
skb->len, PCI_DMA_TODEVICE);
if (pci_dma_mapping_error(priv->pdev, skb_paddr)) {
pr_err("skb DMA mapping error: %pad\n", &skb_paddr);
ret = -ENOMEM;
goto tx_done;
}
txbd = &ps->tx_bd_vbase[i];
txbd->addr = cpu_to_le32(QTN_HOST_LO32(skb_paddr));
txbd->addr_h = cpu_to_le32(QTN_HOST_HI32(skb_paddr));
info = (len & QTN_PCIE_TX_DESC_LEN_MASK) << QTN_PCIE_TX_DESC_LEN_SHIFT;
txbd->info = cpu_to_le32(info);
/* sync up all descriptor updates before passing them to EP */
dma_wmb();
/* write new TX descriptor to PCIE_RX_FIFO on EP */
txbd_paddr = ps->tx_bd_pbase + i * sizeof(struct qtnf_pearl_tx_bd);
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
writel(QTN_HOST_HI32(txbd_paddr),
PCIE_HDP_HOST_WR_DESC0_H(ps->pcie_reg_base));
#endif
writel(QTN_HOST_LO32(txbd_paddr),
PCIE_HDP_HOST_WR_DESC0(ps->pcie_reg_base));
if (++i >= priv->tx_bd_num)
i = 0;
priv->tx_bd_w_index = i;
tx_done:
if (ret && skb) {
pr_err_ratelimited("drop skb\n");
if (skb->dev)
skb->dev->stats.tx_dropped++;
dev_kfree_skb_any(skb);
}
priv->tx_done_count++;
spin_unlock_irqrestore(&priv->tx_lock, flags);
qtnf_pearl_data_tx_reclaim(ps);
return NETDEV_TX_OK;
}
static bool mlx4_en_build_dma_wqe(struct mlx4_en_priv *priv,
struct skb_shared_info *shinfo,
struct mlx4_wqe_data_seg *data,
struct sk_buff *skb,
int lso_header_size,
__be32 mr_key,
struct mlx4_en_tx_info *tx_info)
{
struct device *ddev = priv->ddev;
dma_addr_t dma = 0;
u32 byte_count = 0;
int i_frag;
/* Map fragments if any */
for (i_frag = shinfo->nr_frags - 1; i_frag >= 0; i_frag--) {
const struct skb_frag_struct *frag;
frag = &shinfo->frags[i_frag];
byte_count = skb_frag_size(frag);
dma = skb_frag_dma_map(ddev, frag,
0, byte_count,
DMA_TO_DEVICE);
if (dma_mapping_error(ddev, dma))
goto tx_drop_unmap;
data->addr = cpu_to_be64(dma);
data->lkey = mr_key;
dma_wmb();
data->byte_count = cpu_to_be32(byte_count);
--data;
}
/* Map linear part if needed */
if (tx_info->linear) {
byte_count = skb_headlen(skb) - lso_header_size;
dma = dma_map_single(ddev, skb->data +
lso_header_size, byte_count,
PCI_DMA_TODEVICE);
if (dma_mapping_error(ddev, dma))
goto tx_drop_unmap;
data->addr = cpu_to_be64(dma);
data->lkey = mr_key;
dma_wmb();
data->byte_count = cpu_to_be32(byte_count);
}
/* tx completion can avoid cache line miss for common cases */
tx_info->map0_dma = dma;
tx_info->map0_byte_count = byte_count;
return true;
tx_drop_unmap:
en_err(priv, "DMA mapping error\n");
while (++i_frag < shinfo->nr_frags) {
++data;
dma_unmap_page(ddev, (dma_addr_t)be64_to_cpu(data->addr),
be32_to_cpu(data->byte_count),
PCI_DMA_TODEVICE);
}
return false;
}