void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring = *pring;
uint32_t x;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
buf_ring_free(ring->br, M_DEVBUF);
if (ring->bf_enabled)
mlx4_bf_free(mdev->dev, &ring->bf);
mlx4_qp_remove(mdev->dev, &ring->qp);
mlx4_qp_free(mdev->dev, &ring->qp);
mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1);
mlx4_en_unmap_buffer(&ring->wqres.buf);
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
for (x = 0; x != ring->size; x++)
bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map);
vfree(ring->tx_info);
mtx_destroy(&ring->tx_lock.m);
mtx_destroy(&ring->comp_lock.m);
bus_dma_tag_destroy(ring->dma_tag);
kfree(ring);
*pring = NULL;
}
/*********************************************************************
*
* Free transmit ring related data structures.
*
**********************************************************************/
void
ixl_free_que_tx(struct ixl_queue *que)
{
struct tx_ring *txr = &que->txr;
struct ixl_tx_buf *buf;
INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me);
for (int i = 0; i < que->num_desc; i++) {
buf = &txr->buffers[i];
if (buf->m_head != NULL) {
bus_dmamap_sync(buf->tag, buf->map,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(buf->tag,
buf->map);
m_freem(buf->m_head);
buf->m_head = NULL;
if (buf->map != NULL) {
bus_dmamap_destroy(buf->tag,
buf->map);
buf->map = NULL;
}
} else if (buf->map != NULL) {
bus_dmamap_unload(buf->tag,
buf->map);
bus_dmamap_destroy(buf->tag,
buf->map);
buf->map = NULL;
}
}
if (txr->br != NULL)
buf_ring_free(txr->br, M_DEVBUF);
if (txr->buffers != NULL) {
free(txr->buffers, M_DEVBUF);
txr->buffers = NULL;
}
if (txr->tx_tag != NULL) {
bus_dma_tag_destroy(txr->tx_tag);
txr->tx_tag = NULL;
}
if (txr->tso_tag != NULL) {
bus_dma_tag_destroy(txr->tso_tag);
txr->tso_tag = NULL;
}
INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me);
return;
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring, u32 size,
u16 stride, int node, int queue_idx)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring;
int tmp;
int err;
ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node);
if (!ring) {
ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL);
if (!ring) {
en_err(priv, "Failed allocating TX ring\n");
return -ENOMEM;
}
}
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
#ifdef CONFIG_RATELIMIT
ring->rl_data.rate_index = 0;
/* User_valid should be false in a rate_limit ring until the
* creation process of the ring is done, after the activation. */
if (queue_idx < priv->native_tx_ring_num)
ring->rl_data.user_valid = true;
else
ring->rl_data.user_valid = false;
#endif
ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS;
ring->inline_thold = min(inline_thold, MAX_INLINE);
mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF);
mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF);
/* Allocate the buf ring */
#ifdef CONFIG_RATELIMIT
if (queue_idx < priv->native_tx_ring_num)
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
M_WAITOK, &ring->tx_lock.m);
else
ring->br = buf_ring_alloc(size / 4, M_DEVBUF, M_WAITOK,
&ring->tx_lock.m);
#else
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
M_WAITOK, &ring->tx_lock.m);
#endif
if (ring->br == NULL) {
en_err(priv, "Failed allocating tx_info ring\n");
return -ENOMEM;
}
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = vmalloc_node(tmp, node);
if (!ring->tx_info) {
ring->tx_info = vmalloc(tmp);
if (!ring->tx_info) {
err = -ENOMEM;
goto err_ring;
}
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node);
if (!ring->bounce_buf) {
ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
if (!ring->bounce_buf) {
err = -ENOMEM;
goto err_info;
}
}
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
/* Allocate HW buffers on provided NUMA node */
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_bounce;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d "
"buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_BF_QP);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_map;
}
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_reserve;
}
ring->qp.event = mlx4_en_sqp_event;
err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
if (err) {
en_dbg(DRV, priv, "working without blueflame (%d)", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
} else
ring->bf_enabled = true;
ring->queue_index = queue_idx;
if (queue_idx < priv->num_tx_rings_p_up )
CPU_SET(queue_idx, &ring->affinity_mask);
*pring = ring;
return 0;
err_reserve:
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_bounce:
kfree(ring->bounce_buf);
err_info:
vfree(ring->tx_info);
err_ring:
buf_ring_free(ring->br, M_DEVBUF);
kfree(ring);
return err;
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring, u32 size,
u16 stride, int node, int queue_idx)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring;
uint32_t x;
int tmp;
int err;
ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node);
if (!ring) {
ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL);
if (!ring) {
en_err(priv, "Failed allocating TX ring\n");
return -ENOMEM;
}
}
/* Create DMA descriptor TAG */
if ((err = -bus_dma_tag_create(
bus_get_dma_tag(mdev->pdev->dev.bsddev),
1, /* any alignment */
0, /* no boundary */
BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */
MLX4_EN_TX_MAX_PAYLOAD_SIZE, /* maxsize */
MLX4_EN_TX_MAX_MBUF_FRAGS, /* nsegments */
MLX4_EN_TX_MAX_MBUF_SIZE, /* maxsegsize */
0, /* flags */
NULL, NULL, /* lockfunc, lockfuncarg */
&ring->dma_tag)))
goto done;
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
ring->inline_thold = MAX(MIN_PKT_LEN, MIN(inline_thold, MAX_INLINE));
mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF);
mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF);
/* Allocate the buf ring */
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
M_WAITOK, &ring->tx_lock.m, queue_idx,
priv->tx_ring_num);
if (ring->br == NULL) {
en_err(priv, "Failed allocating tx_info ring\n");
err = -ENOMEM;
goto err_free_dma_tag;
}
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node);
if (!ring->tx_info) {
ring->tx_info = kzalloc(tmp, GFP_KERNEL);
if (!ring->tx_info) {
err = -ENOMEM;
goto err_ring;
}
}
/* Create DMA descriptor MAPs */
for (x = 0; x != size; x++) {
err = -bus_dmamap_create(ring->dma_tag, 0,
&ring->tx_info[x].dma_map);
if (err != 0) {
while (x--) {
bus_dmamap_destroy(ring->dma_tag,
ring->tx_info[x].dma_map);
}
goto err_info;
}
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
/* Allocate HW buffers on provided NUMA node */
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_dma_map;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d "
"buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_BF_QP);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_map;
}
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_reserve;
}
ring->qp.event = mlx4_en_sqp_event;
err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
if (err) {
en_dbg(DRV, priv, "working without blueflame (%d)", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
} else
ring->bf_enabled = true;
ring->queue_index = queue_idx;
if (queue_idx < priv->num_tx_rings_p_up )
CPU_SET(queue_idx, &ring->affinity_mask);
*pring = ring;
return 0;
err_reserve:
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_dma_map:
for (x = 0; x != size; x++)
bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map);
err_info:
vfree(ring->tx_info);
err_ring:
buf_ring_free(ring->br, M_DEVBUF);
err_free_dma_tag:
bus_dma_tag_destroy(ring->dma_tag);
done:
kfree(ring);
return err;
}
