int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring, u32 size)
{
struct mlx4_en_dev *mdev = priv->mdev;
int err;
int tmp;
ring->prod = 0;
ring->cons = 0;
ring->size = size;
ring->size_mask = size - 1;
ring->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
DS_SIZE * MLX4_EN_MAX_RX_FRAGS);
ring->log_stride = ffs(ring->stride) - 1;
ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
sizeof(struct mbuf *));
ring->rx_info = kmalloc(tmp, GFP_KERNEL);
if (!ring->rx_info) {
en_err(priv, "Failed allocating rx_info ring\n");
return -ENOMEM;
}
en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d stride:%d (%d)\n",
ring->rx_info, tmp, ring->stride, ring->log_stride);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
ring->buf_size, 2 * PAGE_SIZE);
if (err)
goto err_ring;
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map RX buffer\n");
goto err_hwq;
}
ring->buf = ring->wqres.buf.direct.buf;
return 0;
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_ring:
kfree(ring->rx_info);
ring->rx_info = NULL;
return err;
}
static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring,
int index)
{
struct mlx4_en_frag_info *frag_info;
struct mlx4_en_dev *mdev = priv->mdev;
struct mbuf **mb_list;
struct mlx4_en_rx_desc *rx_desc = ring->buf + (index << ring->log_stride);
dma_addr_t dma;
int nr;
mb_list = ring->rx_info + (index << priv->log_rx_info);
for (nr = 0; nr < priv->num_frags; nr++) {
en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
frag_info = &priv->frag_info[nr];
dma = be64_to_cpu(rx_desc->data[nr].addr);
en_dbg(DRV, priv, "Unmaping buffer at dma:0x%llx\n", (long long) dma);
pci_unmap_single(mdev->pdev, dma, frag_info->frag_size,
PCI_DMA_FROMDEVICE);
m_free(mb_list[nr]);
}
}
static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring)
{
struct mlx4_en_rx_alloc *page_alloc;
int i;
for (i = 0; i < priv->num_frags; i++) {
page_alloc = &ring->page_alloc[i];
en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
i, page_count(page_alloc->page));
put_page(page_alloc->page);
page_alloc->page = NULL;
}
}
void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring)
{
struct mlx4_en_dev *mdev = priv->mdev;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
mlx4_qp_remove(mdev->dev, &ring->qp);
mlx4_qp_free(mdev->dev, &ring->qp);
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
mlx4_en_unmap_buffer(&ring->wqres.buf);
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
vfree(ring->tx_info);
ring->tx_info = NULL;
}
static inline int invalid_cqe(struct mlx4_en_priv *priv,
struct mlx4_cqe *cqe)
{
/* Drop packet on bad receive or bad checksum */
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
en_err(priv, "CQE completed in error - vendor "
"syndrom:%d syndrom:%d\n",
((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome,
((struct mlx4_err_cqe *) cqe)->syndrome);
return 1;
}
if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
return 1;
}
return 0;
}
void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring = *pring;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
if (ring->bf_alloced)
mlx4_bf_free(mdev->dev, &ring->bf);
mlx4_qp_remove(mdev->dev, &ring->sp_qp);
mlx4_qp_free(mdev->dev, &ring->sp_qp);
mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1);
mlx4_free_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
kvfree(ring->tx_info);
ring->tx_info = NULL;
kfree(ring);
*pring = NULL;
}
void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring)
{
struct mlx4_en_dev *mdev = priv->mdev;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
#ifdef __VMKERNEL_BF_ENABLE__
if (ring->bf_enabled)
mlx4_bf_free(mdev->dev, &ring->bf);
#endif /* __VMKERNEL_BF_ENABLE__ */
mlx4_qp_remove(mdev->dev, &ring->qp);
mlx4_qp_free(mdev->dev, &ring->qp);
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
mlx4_en_unmap_buffer(&ring->wqres.buf);
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
vfree(ring->tx_info);
ring->tx_info = NULL;
}
void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring = *pring;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
buf_ring_free(ring->br, M_DEVBUF);
if (ring->bf_enabled)
mlx4_bf_free(mdev->dev, &ring->bf);
mlx4_qp_remove(mdev->dev, &ring->qp);
mlx4_qp_free(mdev->dev, &ring->qp);
mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1);
mlx4_en_unmap_buffer(&ring->wqres.buf);
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
kfree(ring->bounce_buf);
vfree(ring->tx_info);
mtx_destroy(&ring->tx_lock.m);
mtx_destroy(&ring->comp_lock.m);
kfree(ring);
*pring = NULL;
}
static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring)
{
struct mlx4_en_rx_alloc *page_alloc;
int i;
for (i = 0; i < priv->num_frags; i++) {
const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
page_alloc = &ring->page_alloc[i];
en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
i, page_count(page_alloc->page));
dma_unmap_page(priv->ddev, page_alloc->dma,
page_alloc->page_size, PCI_DMA_FROMDEVICE);
while (page_alloc->page_offset + frag_info->frag_stride <
page_alloc->page_size) {
put_page(page_alloc->page);
page_alloc->page_offset += frag_info->frag_stride;
}
page_alloc->page = NULL;
}
}
static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring)
{
int i;
struct mlx4_en_rx_alloc *page_alloc;
for (i = 0; i < priv->num_frags; i++) {
const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
if (mlx4_alloc_pages(priv, &ring->page_alloc[i],
frag_info, GFP_KERNEL | __GFP_COLD))
goto out;
en_dbg(DRV, priv, " frag %d allocator: - size:%d frags:%d\n",
i, ring->page_alloc[i].page_size,
page_ref_count(ring->page_alloc[i].page));
}
return 0;
out:
while (i--) {
struct page *page;
page_alloc = &ring->page_alloc[i];
dma_unmap_page(priv->ddev, page_alloc->dma,
page_alloc->page_size,
priv->frag_info[i].dma_dir);
page = page_alloc->page;
/* Revert changes done by mlx4_alloc_pages */
page_ref_sub(page, page_alloc->page_size /
priv->frag_info[i].frag_stride - 1);
put_page(page);
page_alloc->page = NULL;
}
return -ENOMEM;
}
/* Allocate rx qp's and configure them according to rss map */
int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_rss_map *rss_map = &priv->rss_map;
struct mlx4_qp_context context;
struct mlx4_en_rss_context *rss_context;
void *ptr;
u8 rss_mask = 0x3f;
int i, qpn;
int err = 0;
int good_qps = 0;
en_dbg(DRV, priv, "Configuring rss steering\n");
err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
priv->rx_ring_num,
&rss_map->base_qpn);
if (err) {
en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
return err;
}
for (i = 0; i < priv->rx_ring_num; i++) {
qpn = rss_map->base_qpn + i;
err = mlx4_en_config_rss_qp(priv, qpn, &priv->rx_ring[i],
&rss_map->state[i],
&rss_map->qps[i]);
if (err)
goto rss_err;
++good_qps;
}
/* Configure RSS indirection qp */
err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp);
if (err) {
en_err(priv, "Failed to allocate RSS indirection QP\n");
goto rss_err;
}
rss_map->indir_qp.event = mlx4_en_sqp_event;
mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
priv->rx_ring[0].cqn, &context);
ptr = ((void *) &context) + 0x3c;
rss_context = (struct mlx4_en_rss_context *) ptr;
rss_context->base_qpn = cpu_to_be32(ilog2(priv->rx_ring_num) << 24 |
(rss_map->base_qpn));
rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
rss_context->flags = rss_mask;
if (priv->mdev->profile.udp_rss)
rss_context->base_qpn_udp = rss_context->default_qpn;
err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
&rss_map->indir_qp, &rss_map->indir_state);
if (err)
goto indir_err;
return 0;
indir_err:
mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
rss_err:
for (i = 0; i < good_qps; i++) {
mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
}
mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
return err;
}
int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cqe *cqe;
struct mlx4_en_rx_ring *ring = &priv->rx_ring[cq->ring];
struct skb_frag_struct *skb_frags;
struct mlx4_en_rx_desc *rx_desc;
struct sk_buff *skb;
int index;
int nr;
unsigned int length;
int polled = 0;
int ip_summed;
if (!priv->port_up)
return 0;
/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
* descriptor offset can be deduced from the CQE index instead of
* reading 'cqe->index' */
index = cq->mcq.cons_index & ring->size_mask;
cqe = &cq->buf[index];
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cq->mcq.cons_index & cq->size)) {
skb_frags = ring->rx_info + (index << priv->log_rx_info);
rx_desc = ring->buf + (index << ring->log_stride);
/*
* make sure we read the CQE after we read the ownership bit
*/
rmb();
/* Drop packet on bad receive or bad checksum */
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
en_err(priv, "CQE completed in error - vendor "
"syndrom:%d syndrom:%d\n",
((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome,
((struct mlx4_err_cqe *) cqe)->syndrome);
goto next;
}
if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
goto next;
}
/*
* Packet is OK - process it.
*/
length = be32_to_cpu(cqe->byte_cnt);
ring->bytes += length;
ring->packets++;
if (likely(dev->features & NETIF_F_RXCSUM)) {
if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
(cqe->checksum == cpu_to_be16(0xffff))) {
priv->port_stats.rx_chksum_good++;
/* This packet is eligible for LRO if it is:
* - DIX Ethernet (type interpretation)
* - TCP/IP (v4)
* - without IP options
* - not an IP fragment */
if (dev->features & NETIF_F_GRO) {
struct sk_buff *gro_skb = napi_get_frags(&cq->napi);
if (!gro_skb)
goto next;
nr = mlx4_en_complete_rx_desc(
priv, rx_desc,
skb_frags, skb_shinfo(gro_skb)->frags,
ring->page_alloc, length);
if (!nr)
goto next;
skb_shinfo(gro_skb)->nr_frags = nr;
gro_skb->len = length;
gro_skb->data_len = length;
gro_skb->truesize += length;
gro_skb->ip_summed = CHECKSUM_UNNECESSARY;
if (priv->vlgrp && (cqe->vlan_my_qpn &
cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK)))
vlan_gro_frags(&cq->napi, priv->vlgrp, be16_to_cpu(cqe->sl_vid));
else
napi_gro_frags(&cq->napi);
goto next;
}
/* LRO not possible, complete processing here */
ip_summed = CHECKSUM_UNNECESSARY;
} else {
ip_summed = CHECKSUM_NONE;
priv->port_stats.rx_chksum_none++;
}
} else {
ip_summed = CHECKSUM_NONE;
priv->port_stats.rx_chksum_none++;
}
skb = mlx4_en_rx_skb(priv, rx_desc, skb_frags,
ring->page_alloc, length);
if (!skb) {
priv->stats.rx_dropped++;
goto next;
}
if (unlikely(priv->validate_loopback)) {
validate_loopback(priv, skb);
goto next;
}
skb->ip_summed = ip_summed;
skb->protocol = eth_type_trans(skb, dev);
skb_record_rx_queue(skb, cq->ring);
/* Push it up the stack */
if (priv->vlgrp && (be32_to_cpu(cqe->vlan_my_qpn) &
MLX4_CQE_VLAN_PRESENT_MASK)) {
vlan_hwaccel_receive_skb(skb, priv->vlgrp,
be16_to_cpu(cqe->sl_vid));
} else
netif_receive_skb(skb);
next:
++cq->mcq.cons_index;
index = (cq->mcq.cons_index) & ring->size_mask;
cqe = &cq->buf[index];
if (++polled == budget) {
/* We are here because we reached the NAPI budget -
* flush only pending LRO sessions */
goto out;
}
}
out:
AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
mlx4_cq_set_ci(&cq->mcq);
wmb(); /* ensure HW sees CQ consumer before we post new buffers */
ring->cons = cq->mcq.cons_index;
ring->prod += polled; /* Polled descriptors were realocated in place */
mlx4_en_update_rx_prod_db(ring);
return polled;
}
static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
struct mlx4_en_rx_desc *rx_desc,
struct skb_frag_struct *skb_frags,
struct mlx4_en_rx_alloc *page_alloc,
unsigned int length)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct sk_buff *skb;
void *va;
int used_frags;
dma_addr_t dma;
skb = dev_alloc_skb(SMALL_PACKET_SIZE + NET_IP_ALIGN);
if (!skb) {
en_dbg(RX_ERR, priv, "Failed allocating skb\n");
return NULL;
}
skb->dev = priv->dev;
skb_reserve(skb, NET_IP_ALIGN);
skb->len = length;
skb->truesize = length + sizeof(struct sk_buff);
/* Get pointer to first fragment so we could copy the headers into the
* (linear part of the) skb */
va = page_address(skb_frags[0].page) + skb_frags[0].page_offset;
if (length <= SMALL_PACKET_SIZE) {
/* We are copying all relevant data to the skb - temporarily
* synch buffers for the copy */
dma = be64_to_cpu(rx_desc->data[0].addr);
dma_sync_single_for_cpu(&mdev->pdev->dev, dma, length,
DMA_FROM_DEVICE);
skb_copy_to_linear_data(skb, va, length);
dma_sync_single_for_device(&mdev->pdev->dev, dma, length,
DMA_FROM_DEVICE);
skb->tail += length;
} else {
/* Move relevant fragments to skb */
used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, skb_frags,
skb_shinfo(skb)->frags,
page_alloc, length);
if (unlikely(!used_frags)) {
kfree_skb(skb);
return NULL;
}
skb_shinfo(skb)->nr_frags = used_frags;
/* Copy headers into the skb linear buffer */
memcpy(skb->data, va, HEADER_COPY_SIZE);
skb->tail += HEADER_COPY_SIZE;
/* Skip headers in first fragment */
skb_shinfo(skb)->frags[0].page_offset += HEADER_COPY_SIZE;
/* Adjust size of first fragment */
skb_shinfo(skb)->frags[0].size -= HEADER_COPY_SIZE;
skb->data_len = length - HEADER_COPY_SIZE;
}
return skb;
}
u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring,
int index, u64 timestamp,
int napi_mode)
{
struct mlx4_en_tx_info *tx_info = &ring->tx_info[index];
struct mlx4_en_rx_alloc frame = {
.page = tx_info->page,
.dma = tx_info->map0_dma,
};
if (!mlx4_en_rx_recycle(ring->recycle_ring, &frame)) {
dma_unmap_page(priv->ddev, tx_info->map0_dma,
PAGE_SIZE, priv->dma_dir);
put_page(tx_info->page);
}
return tx_info->nr_txbb;
}
int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
int cnt = 0;
/* Skip last polled descriptor */
ring->cons += ring->last_nr_txbb;
en_dbg(DRV, priv, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
ring->cons, ring->prod);
if ((u32) (ring->prod - ring->cons) > ring->size) {
if (netif_msg_tx_err(priv))
en_warn(priv, "Tx consumer passed producer!\n");
return 0;
}
while (ring->cons != ring->prod) {
ring->last_nr_txbb = ring->free_tx_desc(priv, ring,
ring->cons & ring->size_mask,
0, 0 /* Non-NAPI caller */);
ring->cons += ring->last_nr_txbb;
cnt++;
}
if (ring->tx_queue)
netdev_tx_reset_queue(ring->tx_queue);
if (cnt)
en_dbg(DRV, priv, "Freed %d uncompleted tx descriptors\n", cnt);
return cnt;
}
bool mlx4_en_process_tx_cq(struct net_device *dev,
struct mlx4_en_cq *cq, int napi_budget)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->type][cq->ring];
struct mlx4_cqe *cqe;
u16 index, ring_index, stamp_index;
u32 txbbs_skipped = 0;
u32 txbbs_stamp = 0;
u32 cons_index = mcq->cons_index;
int size = cq->size;
u32 size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
u32 packets = 0;
u32 bytes = 0;
int factor = priv->cqe_factor;
int done = 0;
int budget = priv->tx_work_limit;
u32 last_nr_txbb;
u32 ring_cons;
if (unlikely(!priv->port_up))
return true;
netdev_txq_bql_complete_prefetchw(ring->tx_queue);
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
last_nr_txbb = READ_ONCE(ring->last_nr_txbb);
ring_cons = READ_ONCE(ring->cons);
ring_index = ring_cons & size_mask;
stamp_index = ring_index;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size) && (done < budget)) {
u16 new_index;
/*
* make sure we read the CQE after we read the
* ownership bit
*/
dma_rmb();
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe;
en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
cqe_err->vendor_err_syndrome,
cqe_err->syndrome);
}
/* Skip over last polled CQE */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
u64 timestamp = 0;
txbbs_skipped += last_nr_txbb;
ring_index = (ring_index + last_nr_txbb) & size_mask;
if (unlikely(ring->tx_info[ring_index].ts_requested))
timestamp = mlx4_en_get_cqe_ts(cqe);
/* free next descriptor */
last_nr_txbb = ring->free_tx_desc(
priv, ring, ring_index,
timestamp, napi_budget);
mlx4_en_stamp_wqe(priv, ring, stamp_index,
!!((ring_cons + txbbs_stamp) &
ring->size));
stamp_index = ring_index;
txbbs_stamp = txbbs_skipped;
packets++;
bytes += ring->tx_info[ring_index].nr_bytes;
} while ((++done < budget) && (ring_index != new_index));
++cons_index;
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
}
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
/* we want to dirty this cache line once */
WRITE_ONCE(ring->last_nr_txbb, last_nr_txbb);
WRITE_ONCE(ring->cons, ring_cons + txbbs_skipped);
if (cq->type == TX_XDP)
return done < budget;
netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
/* Wakeup Tx queue if this stopped, and ring is not full.
*/
if (netif_tx_queue_stopped(ring->tx_queue) &&
!mlx4_en_is_tx_ring_full(ring)) {
netif_tx_wake_queue(ring->tx_queue);
ring->wake_queue++;
}
return done < budget;
}
void mlx4_en_tx_irq(struct mlx4_cq *mcq)
{
struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
struct mlx4_en_priv *priv = netdev_priv(cq->dev);
if (likely(priv->port_up))
napi_schedule_irqoff(&cq->napi);
else
mlx4_en_arm_cq(priv, cq);
}
/* TX CQ polling - called by NAPI */
int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget)
{
struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
struct net_device *dev = cq->dev;
struct mlx4_en_priv *priv = netdev_priv(dev);
bool clean_complete;
clean_complete = mlx4_en_process_tx_cq(dev, cq, budget);
if (!clean_complete)
return budget;
napi_complete(napi);
mlx4_en_arm_cq(priv, cq);
return 0;
}
static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring,
u32 index,
unsigned int desc_size)
{
u32 copy = (ring->size - index) << LOG_TXBB_SIZE;
int i;
for (i = desc_size - copy - 4; i >= 0; i -= 4) {
if ((i & (TXBB_SIZE - 1)) == 0)
wmb();
*((u32 *) (ring->buf + i)) =
*((u32 *) (ring->bounce_buf + copy + i));
}
for (i = copy - 4; i >= 4 ; i -= 4) {
if ((i & (TXBB_SIZE - 1)) == 0)
wmb();
*((u32 *)(ring->buf + (index << LOG_TXBB_SIZE) + i)) =
*((u32 *) (ring->bounce_buf + i));
}
/* Return real descriptor location */
return ring->buf + (index << LOG_TXBB_SIZE);
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring, int qpn, u32 size,
u16 stride)
{
struct mlx4_en_dev *mdev = priv->mdev;
int tmp;
int err;
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
inline_thold = min(inline_thold, MAX_INLINE);
spin_lock_init(&ring->comp_lock);
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = vmalloc(tmp);
if (!ring->tx_info) {
en_err(priv, "Failed allocating tx_info ring\n");
return -ENOMEM;
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
if (!ring->bounce_buf) {
en_err(priv, "Failed allocating bounce buffer\n");
err = -ENOMEM;
goto err_tx;
}
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_bounce;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d "
"buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
ring->qpn = qpn;
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_map;
}
ring->qp.event = mlx4_en_sqp_event;
err = mlx4_bf_alloc(mdev->dev, &ring->bf);
if (err) {
en_dbg(DRV, priv, "working without blueflame (%d)", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
} else
ring->bf_enabled = true;
return 0;
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_bounce:
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
err_tx:
vfree(ring->tx_info);
ring->tx_info = NULL;
return err;
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring, uint32_t size,
uint16_t stride, int node, int queue_index)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring;
int tmp;
int err;
ring = kzalloc_node(sizeof(*ring), KMALLOC_WAIT, node);
if (!ring) {
ring = kzmalloc(sizeof(*ring), KMALLOC_WAIT);
if (!ring) {
en_err(priv, "Failed allocating TX ring\n");
return -ENOMEM;
}
}
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = kmalloc_node(tmp, KMALLOC_WAIT | __GFP_NOWARN, node);
if (!ring->tx_info) {
ring->tx_info = vmalloc(tmp);
if (!ring->tx_info) {
err = -ENOMEM;
goto err_ring;
}
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, KMALLOC_WAIT, node);
if (!ring->bounce_buf) {
ring->bounce_buf = kmalloc(MAX_DESC_SIZE, KMALLOC_WAIT);
if (!ring->bounce_buf) {
err = -ENOMEM;
goto err_info;
}
}
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
/* Allocate HW buffers on provided NUMA node */
set_dev_node(&mdev->dev->persist->pdev->dev, node);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_bounce;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
ring, ring->buf, ring->size, ring->buf_size,
(unsigned long long) ring->wqres.buf.direct.map);
err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_ETH_BF_QP);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_map;
}
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp, KMALLOC_WAIT);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_reserve;
}
ring->qp.event = mlx4_en_sqp_event;
#if 0 // AKAROS_PORT
err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
if (err) {
#else
if (true) {
#endif
en_dbg(DRV, priv, "working without blueflame (%d)\n", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
ring->bf_alloced = false;
priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME;
} else {
ring->bf_alloced = true;
ring->bf_enabled = !!(priv->pflags &
MLX4_EN_PRIV_FLAGS_BLUEFLAME);
}
ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type;
ring->queue_index = queue_index;
if (queue_index < priv->num_tx_rings_p_up)
cpumask_set_cpu(cpumask_local_spread(queue_index,
priv->mdev->dev->numa_node),
&ring->affinity_mask);
*pring = ring;
return 0;
err_reserve:
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_bounce:
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
err_info:
#if 0 // AKAROS_PORT
kvfree(ring->tx_info);
#endif
ring->tx_info = NULL;
err_ring:
kfree(ring);
*pring = NULL;
return err;
}
void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring)
{
panic("Disabled");
#if 0 // AKAROS_PORT
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring = *pring;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
if (ring->bf_alloced)
mlx4_bf_free(mdev->dev, &ring->bf);
mlx4_qp_remove(mdev->dev, &ring->qp);
mlx4_qp_free(mdev->dev, &ring->qp);
mlx4_en_unmap_buffer(&ring->wqres.buf);
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
kvfree(ring->tx_info);
ring->tx_info = NULL;
kfree(ring);
*pring = NULL;
#endif
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring, u32 size,
u16 stride, int node, int queue_index)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring;
int tmp;
int err;
ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node);
if (!ring) {
en_err(priv, "Failed allocating TX ring\n");
return -ENOMEM;
}
ring->size = size;
ring->size_mask = size - 1;
ring->sp_stride = stride;
ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS;
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = kvmalloc_node(tmp, GFP_KERNEL, node);
if (!ring->tx_info) {
err = -ENOMEM;
goto err_ring;
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node);
if (!ring->bounce_buf) {
ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
if (!ring->bounce_buf) {
err = -ENOMEM;
goto err_info;
}
}
ring->buf_size = ALIGN(size * ring->sp_stride, MLX4_EN_PAGE_SIZE);
/* Allocate HW buffers on provided NUMA node */
set_dev_node(&mdev->dev->persist->pdev->dev, node);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_bounce;
}
ring->buf = ring->sp_wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
ring, ring->buf, ring->size, ring->buf_size,
(unsigned long long) ring->sp_wqres.buf.direct.map);
err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_ETH_BF_QP,
MLX4_RES_USAGE_DRIVER);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_hwq_res;
}
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->sp_qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_reserve;
}
ring->sp_qp.event = mlx4_en_sqp_event;
err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
if (err) {
en_dbg(DRV, priv, "working without blueflame (%d)\n", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
ring->bf_alloced = false;
priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME;
} else {
ring->bf_alloced = true;
ring->bf_enabled = !!(priv->pflags &
MLX4_EN_PRIV_FLAGS_BLUEFLAME);
}
ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type;
ring->queue_index = queue_index;
if (queue_index < priv->num_tx_rings_p_up)
cpumask_set_cpu(cpumask_local_spread(queue_index,
priv->mdev->dev->numa_node),
&ring->sp_affinity_mask);
*pring = ring;
return 0;
err_reserve:
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->sp_wqres, ring->buf_size);
err_bounce:
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
err_info:
kvfree(ring->tx_info);
ring->tx_info = NULL;
err_ring:
kfree(ring);
*pring = NULL;
return err;
}
static void mlx4_en_create_rl_res(struct mlx4_en_priv *priv,
int ring_id, u8 rate_index)
{
struct mlx4_en_cq *cq;
struct mlx4_en_tx_ring *tx_ring;
struct mlx4_en_dev *mdev = priv->mdev;
int err = 0;
int node = 0;
int j;
if (priv->tx_ring[ring_id]) {
/* Ring already exists, needs activation */
/* Make sure drbr queue has no left overs from before */
tx_ring = priv->tx_ring[ring_id];
goto activate;
}
err = mlx4_en_create_cq(priv, &priv->tx_cq[ring_id],
MLX4_EN_DEF_RL_TX_RING_SIZE, ring_id, TX, node);
if (err) {
en_err(priv, "Failed to create rate limit tx CQ, ring index %u, rate %u\n", ring_id, rate_index);
goto err_create_cq;
}
err = mlx4_en_create_tx_ring(priv, &priv->tx_ring[ring_id],
MLX4_EN_DEF_RL_TX_RING_SIZE, TXBB_SIZE, node, ring_id);
if (err) {
en_err(priv, "Failed to create rate limited tx ring %u, rate %u\n", ring_id, rate_index);
goto err_create_ring;
}
tx_ring = priv->tx_ring[ring_id];
activate:
sysctl_ctx_init(&tx_ring->rl_data.rl_stats_ctx);
tx_ring->rl_data.rate_index = rate_index;
/* Default moderation */
cq = priv->tx_cq[ring_id];
cq->moder_cnt = priv->tx_frames;
cq->moder_time = priv->tx_usecs;
mutex_lock(&mdev->state_lock);
if (!priv->port_up) {
/* No need activating resources, start_port will take care of that */
tx_ring->rl_data.user_valid = true;
mutex_unlock(&mdev->state_lock);
return;
}
/* Activate resources */
err = mlx4_en_activate_cq(priv, cq, ring_id);
if (err) {
en_err(priv, "Failed activating Rate Limit Tx CQ\n");
goto err_activate_resources;
}
err = mlx4_en_set_cq_moder(priv, cq);
if (err) {
en_err(priv, "Failed setting cq moderation parameters");
mlx4_en_deactivate_cq(priv, cq);
goto err_activate_resources;
}
en_dbg(DRV, priv, "Resetting index of CQ:%d to -1\n", ring_id);
cq->buf->wqe_index = cpu_to_be16(0xffff);
err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn,
MLX4_EN_DEF_RL_USER_PRIO);
if (err) {
en_err(priv, "Failed activating rate limit TX ring\n");
mlx4_en_deactivate_cq(priv, cq);
goto err_activate_resources;
}
/* Arm CQ for TX completions */
mlx4_en_arm_cq(priv, cq);
/* Set initial ownership of all Tx TXBBs to SW (1) */
for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE)
*((u32 *) (tx_ring->buf + j)) = INIT_OWNER_BIT;
/* Set ring as valid */
tx_ring->rl_data.user_valid = true;
mutex_unlock(&mdev->state_lock);
priv->rate_limit_tx_ring_num++;
/* Add rate limit statistics to sysctl if debug option was enabled */
if (show_rl_sysctl_info)
mlx4_en_rate_limit_sysctl_stat(priv, ring_id);
return;
err_activate_resources:
mlx4_en_invalidate_rl_ring(priv, ring_id);
mlx4_en_rl_reused_index_insert(priv, ring_id);
atomic_subtract_int(&priv->rate_limits[rate_index].ref, 1);
mutex_unlock(&mdev->state_lock);
return;
err_create_ring:
if (priv->tx_cq[ring_id])
mlx4_en_destroy_cq(priv, &priv->tx_cq[ring_id]);
err_create_cq:
mlx4_en_rl_reused_index_insert(priv, ring_id);
atomic_subtract_int(&priv->rate_limits[rate_index].ref, 1);
}
void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
int is_tx, int rss, int qpn, int cqn,
#ifdef HAVE_NEW_TX_RING_SCHEME
int user_prio, struct mlx4_qp_context *context,
#else
struct mlx4_qp_context *context,
#endif
int idx)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct net_device *dev = priv->dev;
memset(context, 0, sizeof *context);
context->flags = cpu_to_be32(MLX4_QP_ST_MLX << 16 | rss << MLX4_RSS_QPC_FLAG_OFFSET);
context->pd = cpu_to_be32(mdev->priv_pdn);
context->mtu_msgmax = 0xff;
if (!is_tx && !rss)
context->rq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4);
if (is_tx) {
context->sq_size_stride = ilog2(size) << 3 | (ilog2(stride) - 4);
if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PORT_REMAP)
context->params2 |= MLX4_QP_BIT_FPP;
} else {
context->sq_size_stride = ilog2(TXBB_SIZE) - 4;
}
context->usr_page =
cpu_to_be32(mlx4_to_hw_uar_index(mdev->dev, mdev->priv_uar.index));
context->local_qpn = cpu_to_be32(qpn);
context->pri_path.ackto = 1 & 0x07;
context->pri_path.sched_queue = 0x83 | (priv->port - 1) << 6;
#ifdef HAVE_NEW_TX_RING_SCHEME
if (user_prio >= 0) {
context->pri_path.sched_queue |= user_prio << 3;
context->pri_path.feup = MLX4_FEUP_FORCE_ETH_UP;
}
#endif
if (idx != MLX4_EN_NO_VLAN) {
context->pri_path.fl |= MLX4_FL_CV;
context->pri_path.vlan_index = idx;
}
context->pri_path.counter_index = (u8)(priv->counter_index);
context->cqn_send = cpu_to_be32(cqn);
context->cqn_recv = cpu_to_be32(cqn);
if (!rss &&
(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK) &&
context->pri_path.counter_index != MLX4_SINK_COUNTER_INDEX) {
/* disable multicast loopback to qp with same counter */
if (!(dev->features & NETIF_F_LOOPBACK))
context->pri_path.fl |= MLX4_FL_ETH_SRC_CHECK_MC_LB;
context->pri_path.control |=
MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER;
}
context->db_rec_addr = cpu_to_be64(priv->res.db.dma << 2);
if (!(dev->features & NETIF_F_HW_VLAN_CTAG_RX))
context->param3 |= cpu_to_be32(1 << 30);
if (!is_tx && !rss &&
(priv->prof->inline_scatter_thold >= MIN_INLINE_SCATTER)) {
context->param3 |= cpu_to_be32(1 << 25);
}
if (!is_tx && !rss &&
(mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)) {
en_dbg(HW, priv, "Setting RX qp %x tunnel mode to RX tunneled & non-tunneled\n", qpn);
context->srqn = cpu_to_be32(7 << 28); /* this fills bits 30:28 */
}
}
int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring, u32 size, u16 stride)
{
struct mlx4_en_dev *mdev = priv->mdev;
int err;
int tmp;
ring->prod = 0;
ring->cons = 0;
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
ring->log_stride = ffs(ring->stride) - 1;
ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
sizeof(struct skb_frag_struct));
ring->rx_info = vmalloc(tmp);
if (!ring->rx_info) {
en_err(priv, "Failed allocating rx_info ring\n");
return -ENOMEM;
}
en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
ring->rx_info, tmp);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
ring->buf_size, 2 * PAGE_SIZE);
if (err)
goto err_ring;
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map RX buffer\n");
goto err_hwq;
}
ring->buf = ring->wqres.buf.direct.buf;
/* Configure lro mngr */
memset(&ring->lro, 0, sizeof(struct net_lro_mgr));
ring->lro.dev = priv->dev;
ring->lro.features = LRO_F_NAPI;
ring->lro.frag_align_pad = NET_IP_ALIGN;
ring->lro.ip_summed = CHECKSUM_UNNECESSARY;
ring->lro.ip_summed_aggr = CHECKSUM_UNNECESSARY;
ring->lro.max_desc = mdev->profile.num_lro;
ring->lro.max_aggr = MAX_SKB_FRAGS;
ring->lro.lro_arr = kzalloc(mdev->profile.num_lro *
sizeof(struct net_lro_desc),
GFP_KERNEL);
if (!ring->lro.lro_arr) {
en_err(priv, "Failed to allocate lro array\n");
goto err_map;
}
ring->lro.get_frag_header = mlx4_en_get_frag_header;
return 0;
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_ring:
vfree(ring->rx_info);
ring->rx_info = NULL;
return err;
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring, u32 size,
u16 stride, int node, int queue_idx)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring;
int tmp;
int err;
ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node);
if (!ring) {
ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL);
if (!ring) {
en_err(priv, "Failed allocating TX ring\n");
return -ENOMEM;
}
}
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
#ifdef CONFIG_RATELIMIT
ring->rl_data.rate_index = 0;
/* User_valid should be false in a rate_limit ring until the
* creation process of the ring is done, after the activation. */
if (queue_idx < priv->native_tx_ring_num)
ring->rl_data.user_valid = true;
else
ring->rl_data.user_valid = false;
#endif
ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS;
ring->inline_thold = min(inline_thold, MAX_INLINE);
mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF);
mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF);
/* Allocate the buf ring */
#ifdef CONFIG_RATELIMIT
if (queue_idx < priv->native_tx_ring_num)
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
M_WAITOK, &ring->tx_lock.m);
else
ring->br = buf_ring_alloc(size / 4, M_DEVBUF, M_WAITOK,
&ring->tx_lock.m);
#else
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
M_WAITOK, &ring->tx_lock.m);
#endif
if (ring->br == NULL) {
en_err(priv, "Failed allocating tx_info ring\n");
return -ENOMEM;
}
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = vmalloc_node(tmp, node);
if (!ring->tx_info) {
ring->tx_info = vmalloc(tmp);
if (!ring->tx_info) {
err = -ENOMEM;
goto err_ring;
}
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node);
if (!ring->bounce_buf) {
ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
if (!ring->bounce_buf) {
err = -ENOMEM;
goto err_info;
}
}
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
/* Allocate HW buffers on provided NUMA node */
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_bounce;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d "
"buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_BF_QP);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_map;
}
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_reserve;
}
ring->qp.event = mlx4_en_sqp_event;
err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
if (err) {
en_dbg(DRV, priv, "working without blueflame (%d)", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
} else
ring->bf_enabled = true;
ring->queue_index = queue_idx;
if (queue_idx < priv->num_tx_rings_p_up )
CPU_SET(queue_idx, &ring->affinity_mask);
*pring = ring;
return 0;
err_reserve:
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_bounce:
kfree(ring->bounce_buf);
err_info:
vfree(ring->tx_info);
err_ring:
buf_ring_free(ring->br, M_DEVBUF);
kfree(ring);
return err;
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring, u32 size,
u16 stride, int node, int queue_idx)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring;
uint32_t x;
int tmp;
int err;
ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node);
if (!ring) {
ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL);
if (!ring) {
en_err(priv, "Failed allocating TX ring\n");
return -ENOMEM;
}
}
/* Create DMA descriptor TAG */
if ((err = -bus_dma_tag_create(
bus_get_dma_tag(mdev->pdev->dev.bsddev),
1, /* any alignment */
0, /* no boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
MLX4_EN_TX_MAX_PAYLOAD_SIZE, /* maxsize */
MLX4_EN_TX_MAX_MBUF_FRAGS, /* nsegments */
MLX4_EN_TX_MAX_MBUF_SIZE, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockfuncarg */
&ring->dma_tag)))
goto done;
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
ring->inline_thold = MAX(MIN_PKT_LEN, MIN(inline_thold, MAX_INLINE));
mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF);
mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF);
/* Allocate the buf ring */
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
M_WAITOK, &ring->tx_lock.m, queue_idx,
priv->tx_ring_num);
if (ring->br == NULL) {
en_err(priv, "Failed allocating tx_info ring\n");
err = -ENOMEM;
goto err_free_dma_tag;
}
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node);
if (!ring->tx_info) {
ring->tx_info = kzalloc(tmp, GFP_KERNEL);
if (!ring->tx_info) {
err = -ENOMEM;
goto err_ring;
}
}
/* Create DMA descriptor MAPs */
for (x = 0; x != size; x++) {
err = -bus_dmamap_create(ring->dma_tag, 0,
&ring->tx_info[x].dma_map);
if (err != 0) {
while (x--) {
bus_dmamap_destroy(ring->dma_tag,
ring->tx_info[x].dma_map);
}
goto err_info;
}
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
/* Allocate HW buffers on provided NUMA node */
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_dma_map;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d "
"buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_BF_QP);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_map;
}
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_reserve;
}
ring->qp.event = mlx4_en_sqp_event;
err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
if (err) {
en_dbg(DRV, priv, "working without blueflame (%d)", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
} else
ring->bf_enabled = true;
ring->queue_index = queue_idx;
if (queue_idx < priv->num_tx_rings_p_up )
CPU_SET(queue_idx, &ring->affinity_mask);
*pring = ring;
return 0;
err_reserve:
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_dma_map:
for (x = 0; x != size; x++)
bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map);
err_info:
vfree(ring->tx_info);
err_ring:
buf_ring_free(ring->br, M_DEVBUF);
err_free_dma_tag:
bus_dma_tag_destroy(ring->dma_tag);
done:
kfree(ring);
return err;
}
