int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
int cq_idx)
{
struct mlx4_en_dev *mdev = priv->mdev;
int err = 0;
char name[25];
int timestamp_en = 0;
bool assigned_eq = false;
cq->dev = mdev->pndev[priv->port];
cq->mcq.set_ci_db = cq->wqres.db.db;
cq->mcq.arm_db = cq->wqres.db.db + 1;
*cq->mcq.set_ci_db = 0;
*cq->mcq.arm_db = 0;
memset(cq->buf, 0, cq->buf_size);
if (cq->is_tx == RX) {
if (!mlx4_is_eq_vector_valid(mdev->dev, priv->port,
cq->vector)) {
cq->vector = cpumask_first(priv->rx_ring[cq->ring]->affinity_mask);
err = mlx4_assign_eq(mdev->dev, priv->port,
MLX4_EQ_ID_TO_UUID(MLX4_EQ_ID_EN,
priv->port,
cq_idx),
mlx4_en_cq_eq_cb,
&priv->rx_cq[cq_idx],
&cq->vector);
if (err) {
mlx4_err(mdev, "Failed assigning an EQ to %s\n",
name);
goto free_eq;
}
assigned_eq = true;
}
/* Set IRQ for specific name (per ring) */
err = mlx4_rename_eq(mdev->dev, priv->port, cq->vector,
MLX4_EN_EQ_NAME_PRIORITY, "%s-%d",
priv->dev->name, cq->ring);
if (err) {
mlx4_warn(mdev, "Failed to rename EQ, continuing with default name\n");
err = 0;
}
#if defined(HAVE_IRQ_DESC_GET_IRQ_DATA) && defined(HAVE_IRQ_TO_DESC_EXPORTED)
cq->irq_desc =
irq_to_desc(mlx4_eq_get_irq(mdev->dev,
cq->vector));
#endif
} else {
/* For TX we use the same irq per
ring we assigned for the RX */
struct mlx4_en_cq *rx_cq;
cq_idx = cq_idx % priv->rx_ring_num;
rx_cq = priv->rx_cq[cq_idx];
cq->vector = rx_cq->vector;
}
if (!cq->is_tx)
cq->size = priv->rx_ring[cq->ring]->actual_size;
if ((cq->is_tx && priv->hwtstamp_config.tx_type) ||
(!cq->is_tx && priv->hwtstamp_config.rx_filter))
timestamp_en = 1;
err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt,
&mdev->priv_uar, cq->wqres.db.dma, &cq->mcq,
cq->vector, 0, timestamp_en, &cq->wqres.buf, false);
if (err)
goto free_eq;
cq->mcq.comp = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq;
cq->mcq.event = mlx4_en_cq_event;
if (cq->is_tx) {
netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_tx_cq,
NAPI_POLL_WEIGHT);
} else {
netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq, 64);
#ifdef HAVE_NAPI_HASH_ADD
napi_hash_add(&cq->napi);
#endif
}
napi_enable(&cq->napi);
return 0;
free_eq:
if (assigned_eq)
mlx4_release_eq(mdev->dev, MLX4_EQ_ID_TO_UUID(
MLX4_EQ_ID_EN, priv->port, cq_idx),
cq->vector);
cq->vector = mdev->dev->caps.num_comp_vectors;
return err;
}
int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
int cq_idx)
{
struct mlx4_en_dev *mdev = priv->mdev;
int err = 0;
char name[25];
int timestamp_en = 0;
struct cpu_rmap *rmap =
#ifdef CONFIG_RFS_ACCEL
priv->dev->rx_cpu_rmap;
#else
NULL;
#endif
cq->dev = mdev->pndev[priv->port];
cq->mcq.set_ci_db = cq->wqres.db.db;
cq->mcq.arm_db = cq->wqres.db.db + 1;
*cq->mcq.set_ci_db = 0;
*cq->mcq.arm_db = 0;
memset(cq->buf, 0, cq->buf_size);
if (cq->is_tx == RX) {
if (mdev->dev->caps.comp_pool) {
if (!cq->vector) {
sprintf(name, "%s-%d", priv->dev->name,
cq->ring);
/* Set IRQ for specific name (per ring) */
if (mlx4_assign_eq(mdev->dev, name, rmap,
&cq->vector)) {
cq->vector = (cq->ring + 1 + priv->port)
% mdev->dev->caps.num_comp_vectors;
mlx4_warn(mdev, "Failed assigning an EQ to %s, falling back to legacy EQ's\n",
name);
}
cq->irq_desc =
irq_to_desc(mlx4_eq_get_irq(mdev->dev,
cq->vector));
}
} else {
cq->vector = (cq->ring + 1 + priv->port) %
mdev->dev->caps.num_comp_vectors;
}
} else {
/* For TX we use the same irq per
ring we assigned for the RX */
struct mlx4_en_cq *rx_cq;
cq_idx = cq_idx % priv->rx_ring_num;
rx_cq = priv->rx_cq[cq_idx];
cq->vector = rx_cq->vector;
}
if (!cq->is_tx)
cq->size = priv->rx_ring[cq->ring]->actual_size;
if ((cq->is_tx && priv->hwtstamp_config.tx_type) ||
(!cq->is_tx && priv->hwtstamp_config.rx_filter))
timestamp_en = 1;
err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt,
&mdev->priv_uar, cq->wqres.db.dma, &cq->mcq,
cq->vector, 0, timestamp_en);
if (err)
return err;
cq->mcq.comp = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq;
cq->mcq.event = mlx4_en_cq_event;
if (cq->is_tx) {
netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_tx_cq,
NAPI_POLL_WEIGHT);
} else {
struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring];
err = irq_set_affinity_hint(cq->mcq.irq,
ring->affinity_mask);
if (err)
mlx4_warn(mdev, "Failed setting affinity hint\n");
netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq, 64);
napi_hash_add(&cq->napi);
}
napi_enable(&cq->napi);
return 0;
}
/**
* ixgbe_alloc_q_vector - Allocate memory for a single interrupt vector
* @adapter: board private structure to initialize
* @v_count: q_vectors allocated on adapter, used for ring interleaving
* @v_idx: index of vector in adapter struct
* @txr_count: total number of Tx rings to allocate
* @txr_idx: index of first Tx ring to allocate
* @rxr_count: total number of Rx rings to allocate
* @rxr_idx: index of first Rx ring to allocate
*
* We allocate one q_vector. If allocation fails we return -ENOMEM.
**/
static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
int v_count, int v_idx,
int txr_count, int txr_idx,
int rxr_count, int rxr_idx)
{
struct ixgbe_q_vector *q_vector;
struct ixgbe_ring *ring;
int node = NUMA_NO_NODE;
int cpu = -1;
int ring_count, size;
u8 tcs = netdev_get_num_tc(adapter->netdev);
ring_count = txr_count + rxr_count;
size = sizeof(struct ixgbe_q_vector) +
(sizeof(struct ixgbe_ring) * ring_count);
/* customize cpu for Flow Director mapping */
if ((tcs <= 1) && !(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {
u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
if (rss_i > 1 && adapter->atr_sample_rate) {
if (cpu_online(v_idx)) {
cpu = v_idx;
node = cpu_to_node(cpu);
}
}
}
/* allocate q_vector and rings */
q_vector = kzalloc_node(size, GFP_KERNEL, node);
if (!q_vector)
q_vector = kzalloc(size, GFP_KERNEL);
if (!q_vector)
return -ENOMEM;
/* setup affinity mask and node */
if (cpu != -1)
cpumask_set_cpu(cpu, &q_vector->affinity_mask);
q_vector->numa_node = node;
#ifdef CONFIG_IXGBE_DCA
/* initialize CPU for DCA */
q_vector->cpu = -1;
#endif
/* initialize NAPI */
netif_napi_add(adapter->netdev, &q_vector->napi,
ixgbe_poll, 64);
napi_hash_add(&q_vector->napi);
/* tie q_vector and adapter together */
adapter->q_vector[v_idx] = q_vector;
q_vector->adapter = adapter;
q_vector->v_idx = v_idx;
/* initialize work limits */
q_vector->tx.work_limit = adapter->tx_work_limit;
/* initialize pointer to rings */
ring = q_vector->ring;
/* intialize ITR */
if (txr_count && !rxr_count) {
/* tx only vector */
if (adapter->tx_itr_setting == 1)
q_vector->itr = IXGBE_10K_ITR;
else
q_vector->itr = adapter->tx_itr_setting;
} else {
/* rx or rx/tx vector */
if (adapter->rx_itr_setting == 1)
q_vector->itr = IXGBE_20K_ITR;
else
q_vector->itr = adapter->rx_itr_setting;
}
while (txr_count) {
/* assign generic ring traits */
ring->dev = &adapter->pdev->dev;
ring->netdev = adapter->netdev;
/* configure backlink on ring */
ring->q_vector = q_vector;
/* update q_vector Tx values */
ixgbe_add_ring(ring, &q_vector->tx);
/* apply Tx specific ring traits */
ring->count = adapter->tx_ring_count;
if (adapter->num_rx_pools > 1)
ring->queue_index =
txr_idx % adapter->num_rx_queues_per_pool;
else
ring->queue_index = txr_idx;
/* assign ring to adapter */
adapter->tx_ring[txr_idx] = ring;
/* update count and index */
txr_count--;
txr_idx += v_count;
/* push pointer to next ring */
ring++;
}
while (rxr_count) {
/* assign generic ring traits */
ring->dev = &adapter->pdev->dev;
ring->netdev = adapter->netdev;
/* configure backlink on ring */
ring->q_vector = q_vector;
/* update q_vector Rx values */
ixgbe_add_ring(ring, &q_vector->rx);
/*
* 82599 errata, UDP frames with a 0 checksum
* can be marked as checksum errors.
*/
if (adapter->hw.mac.type == ixgbe_mac_82599EB)
set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state);
#ifdef IXGBE_FCOE
if (adapter->netdev->features & NETIF_F_FCOE_MTU) {
struct ixgbe_ring_feature *f;
f = &adapter->ring_feature[RING_F_FCOE];
if ((rxr_idx >= f->offset) &&
(rxr_idx < f->offset + f->indices))
set_bit(__IXGBE_RX_FCOE, &ring->state);
}
#endif /* IXGBE_FCOE */
/* apply Rx specific ring traits */
ring->count = adapter->rx_ring_count;
if (adapter->num_rx_pools > 1)
ring->queue_index =
rxr_idx % adapter->num_rx_queues_per_pool;
else
ring->queue_index = rxr_idx;
/* assign ring to adapter */
adapter->rx_ring[rxr_idx] = ring;
/* update count and index */
rxr_count--;
rxr_idx += v_count;
/* push pointer to next ring */
ring++;
}
return 0;
}
static int sn_alloc_queues(struct sn_device *dev,
void *rings, uint64_t rings_size,
struct tx_queue_opts *txq_opts,
struct rx_queue_opts *rxq_opts)
{
struct sn_queue *queue;
char *p = rings;
void *memchunk;
int num_queues;
int i;
int ret;
ret = netif_set_real_num_tx_queues(dev->netdev, dev->num_txq);
if (ret) {
log_err("netif_set_real_num_tx_queues() failed\n");
return ret;
}
ret = netif_set_real_num_rx_queues(dev->netdev, dev->num_rxq);
if (ret) {
log_err("netif_set_real_num_rx_queues() failed\n");
return ret;
}
num_queues = dev->num_txq + dev->num_rxq;
memchunk = kzalloc(sizeof(struct sn_queue) * num_queues, GFP_KERNEL);
if (!memchunk)
return -ENOMEM;
queue = memchunk;
for (i = 0; i < dev->num_txq; i++) {
dev->tx_queues[i] = queue;
queue->dev = dev;
queue->queue_id = i;
queue->tx.opts = *txq_opts;
queue->tx.netdev_txq = netdev_get_tx_queue(dev->netdev, i);
queue->drv_to_sn = (struct llring *)p;
p += llring_bytes(queue->drv_to_sn);
queue->sn_to_drv = (struct llring *)p;
p += llring_bytes(queue->sn_to_drv);
queue++;
}
for (i = 0; i < dev->num_rxq; i++) {
dev->rx_queues[i] = queue;
queue->dev = dev;
queue->queue_id = i;
queue->rx.opts = *rxq_opts;
queue->rx.rx_regs = (struct sn_rxq_registers *)p;
p += sizeof(struct sn_rxq_registers);
queue->drv_to_sn = (struct llring *)p;
p += llring_bytes(queue->drv_to_sn);
queue->sn_to_drv = (struct llring *)p;
p += llring_bytes(queue->sn_to_drv);
queue++;
}
if ((uintptr_t)p != (uintptr_t)rings + rings_size) {
log_err("Invalid ring space size: %llu, not %llu, at%p)\n",
rings_size,
(uint64_t)((uintptr_t)p - (uintptr_t)rings),
rings);
kfree(memchunk);
return -EFAULT;
}
for (i = 0; i < dev->num_rxq; i++) {
netif_napi_add(dev->netdev, &dev->rx_queues[i]->rx.napi,
sn_poll, NAPI_POLL_WEIGHT);
#ifdef CONFIG_NET_RX_BUSY_POLL
napi_hash_add(&dev->rx_queues[i]->rx.napi);
#endif
spin_lock_init(&dev->rx_queues[i]->rx.lock);
}
sn_test_cache_alignment(dev);
return 0;
}
static int sn_alloc_queues(struct sn_device *dev, void *rings, u64 rings_size)
{
struct sn_queue *queue;
char *p = rings;
void *memchunk;
int num_queues;
int i;
int ret;
ret = netif_set_real_num_tx_queues(dev->netdev, dev->num_txq);
if (ret) {
log_err("netif_set_real_num_tx_queues() failed\n");
return ret;
}
ret = netif_set_real_num_rx_queues(dev->netdev, dev->num_rxq);
if (ret) {
log_err("netif_set_real_num_rx_queues() failed\n");
return ret;
}
num_queues = dev->num_txq + dev->num_rxq;
memchunk = kzalloc(sizeof(struct sn_queue) * num_queues, GFP_KERNEL);
if (!memchunk)
return -ENOMEM;
queue = memchunk;
for (i = 0; i < dev->num_txq; i++) {
dev->tx_queues[i] = queue;
queue->dev = dev;
queue->queue_id = i;
queue->is_rx = false;
queue->drv_to_sn = (struct llring *)p;
p += llring_bytes(queue->drv_to_sn);
queue->sn_to_drv = (struct llring *)p;
p += llring_bytes(queue->sn_to_drv);
queue++;
}
for (i = 0; i < dev->num_rxq; i++) {
dev->rx_queues[i] = queue;
queue->dev = dev;
queue->queue_id = i;
queue->is_rx = true;
sn_stack_init(&queue->ready_tx_meta);
queue->rx_regs = (struct sn_rxq_registers *)p;
p += sizeof(struct sn_rxq_registers);
queue->drv_to_sn = (struct llring *)p;
p += llring_bytes(queue->drv_to_sn);
queue->sn_to_drv = (struct llring *)p;
p += llring_bytes(queue->sn_to_drv);
queue++;
}
if ((uint64_t)p != (uint64_t)rings + rings_size) {
log_err("Invalid ring space size: %llu, not %llu, at%p)\n",
rings_size,
(uint64_t)p - (uint64_t)rings,
rings);
kfree(memchunk);
return -EFAULT;
}
for (i = 0; i < dev->num_rxq; i++) {
netif_napi_add(dev->netdev, &dev->rx_queues[i]->napi,
sn_poll, NAPI_POLL_WEIGHT);
napi_hash_add(&dev->rx_queues[i]->napi);
spin_lock_init(&dev->rx_queues[i]->lock);
}
sn_test_cache_alignment(dev);
return 0;
}