/* Check whether the requested rate is valid.
* If so, retrieve the relevant rate index. */
static int mlx4_en_validate_rate_ctl_req(struct mlx4_en_priv *priv,
struct ifreq_hwtxring *rl_req, u8 *rate_index)
{
int i;
u32 rate;
/* Kernel passes rate in bytes and the driver converts it to bits in order
* to communicate with the hardware. */
rl_req->txringid_max_rate = rl_req->txringid_max_rate * BITS_PER_BYTE;
rate = rl_req->txringid_max_rate;
if (rate > priv->mdev->dev->caps.rl_caps.calc_max_val ||
(rate < priv->mdev->dev->caps.rl_caps.calc_min_val &&
rate != 0)) {
en_err(priv, "Not valid rate limit : %u Bps %d\n",rate / BITS_PER_BYTE, priv->port);
return (EINVAL);
}
/* Searching for the requested rate in the rate table */
for (i = 0; i <= priv->num_rates_per_prio; i++) {
if (priv->rate_limits[i].rate == rate) {
*rate_index = i;
return (0);
}
}
en_err(priv, "Not existing rate limit %u Bps %d\n",rate / BITS_PER_BYTE, priv->port);
return (EINVAL);
}
static int mlx4_en_ets_validate(struct mlx4_en_priv *priv, struct ieee_ets *ets)
{
int i;
int total_ets_bw = 0;
int has_ets_tc = 0;
for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
if (ets->prio_tc[i] > MLX4_EN_NUM_UP) {
en_err(priv, "Bad priority in UP <=> TC mapping. TC: %d, UP: %d\n",
i, ets->prio_tc[i]);
return -EINVAL;
}
switch (ets->tc_tsa[i]) {
case IEEE_8021QAZ_TSA_STRICT:
break;
case IEEE_8021QAZ_TSA_ETS:
has_ets_tc = 1;
total_ets_bw += ets->tc_tx_bw[i];
break;
default:
en_err(priv, "TC[%d]: Not supported TSA: %d\n",
i, ets->tc_tsa[i]);
return -ENOTSUPP;
}
}
if (has_ets_tc && total_ets_bw != MLX4_EN_BW_MAX) {
en_err(priv, "Bad ETS BW sum: %d. Should be exactly 100%%\n",
total_ets_bw);
return -EINVAL;
}
return 0;
}
int mlx4_en_create_rate_limit_ring(struct mlx4_en_priv *priv,
struct ifreq_hwtxring *rl_req)
{
int err = 0;
int index = 0;
u8 rate_index;
/* Check for HW/FW support */
if (!priv->mdev->dev->caps.rl_caps.enable) {
en_err(priv, "No HW/FW support for rate limit rings\n");
return (ENODEV);
}
/* Validate rate limit request */
if(mlx4_en_validate_rate_ctl_req(priv, rl_req, &rate_index))
return (EINVAL);
/* Find available ring index */
index = mlx4_en_find_available_tx_ring_index(priv);
if (index < 0) {
en_err(priv, "Failed to create Rate limit resources, "
"Max capacity reached\n");
return (EINVAL);
}
atomic_add_int(&priv->rate_limits[rate_index].ref, 1);
rl_req->txringid = index;
/* Defer ring creation */
err = mlx4_en_defer_rl_op(priv, rl_req->txringid, rate_index, MLX4_EN_RL_ADD);
return err;
}
static int mlx4_en_test_loopback(struct mlx4_en_priv *priv)
{
u32 loopback_ok = 0;
int i;
priv->loopback_ok = 0;
priv->validate_loopback = 1;
/* xmit */
if (mlx4_en_test_loopback_xmit(priv)) {
en_err(priv, "Transmitting loopback packet failed\n");
goto mlx4_en_test_loopback_exit;
}
/* polling for result */
for (i = 0; i < MLX4_EN_LOOPBACK_RETRIES; ++i) {
msleep(MLX4_EN_LOOPBACK_TIMEOUT);
if (priv->loopback_ok) {
loopback_ok = 1;
break;
}
}
if (!loopback_ok)
en_err(priv, "Loopback packet didn't arrive\n");
mlx4_en_test_loopback_exit:
priv->validate_loopback = 0;
return !loopback_ok;
}
static void mlx4_en_modify_rl_res(struct mlx4_en_priv *priv,
int ring_id, u8 rate_index)
{
struct mlx4_en_tx_ring *tx_ring;
struct mlx4_update_qp_params update_params;
int err;
tx_ring = priv->tx_ring[ring_id];
/* Ring validation */
if(!TX_RING_USER_VALID(ring_id)) {
en_err(priv, "Failed modifying new rate, ring %d doesn't exist\n", ring_id);
/* If the modified ring does not exist, no need to add one
* to the reference count of the requested rate */
atomic_subtract_int(&priv->rate_limits[rate_index].ref, 1);
return;
}
if (priv->rate_limits[tx_ring->rl_data.rate_index].rate !=
priv->rate_limits[rate_index].rate) {
update_params.rl_index = rate_index;
err = mlx4_update_qp(priv->mdev->dev, tx_ring->qpn, MLX4_UPDATE_QP_RATE_LIMIT,
&update_params);
if (err) {
en_err(priv, "Failed updating ring %d with new rate %uBytes/sec, err: %d\n",
ring_id, (priv->rate_limits[rate_index].rate/8), err);
atomic_subtract_int(&priv->rate_limits[rate_index].ref, 1);
return;
}
}
atomic_subtract_int(&priv->rate_limits[tx_ring->rl_data.rate_index].ref, 1);
tx_ring->rl_data.rate_index = rate_index;
}
int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring, u32 size)
{
struct mlx4_en_dev *mdev = priv->mdev;
int err;
int tmp;
ring->prod = 0;
ring->cons = 0;
ring->size = size;
ring->size_mask = size - 1;
ring->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
DS_SIZE * MLX4_EN_MAX_RX_FRAGS);
ring->log_stride = ffs(ring->stride) - 1;
ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
sizeof(struct mbuf *));
ring->rx_info = kmalloc(tmp, GFP_KERNEL);
if (!ring->rx_info) {
en_err(priv, "Failed allocating rx_info ring\n");
return -ENOMEM;
}
en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d stride:%d (%d)\n",
ring->rx_info, tmp, ring->stride, ring->log_stride);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
ring->buf_size, 2 * PAGE_SIZE);
if (err)
goto err_ring;
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map RX buffer\n");
goto err_hwq;
}
ring->buf = ring->wqres.buf.direct.buf;
return 0;
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_ring:
kfree(ring->rx_info);
ring->rx_info = NULL;
return err;
}
static int mlx4_en_dcbnl_ieee_setpfc(struct net_device *dev,
struct ieee_pfc *pfc)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_en_dev *mdev = priv->mdev;
int err;
en_dbg(DRV, priv, "cap: 0x%x en: 0x%x mbc: 0x%x delay: %d\n",
pfc->pfc_cap,
pfc->pfc_en,
pfc->mbc,
pfc->delay);
priv->prof->rx_pause = priv->prof->tx_pause = !!pfc->pfc_en;
priv->prof->rx_ppp = priv->prof->tx_ppp = pfc->pfc_en;
err = mlx4_SET_PORT_general(mdev->dev, priv->port,
priv->rx_skb_size + ETH_FCS_LEN,
priv->prof->tx_pause,
priv->prof->tx_ppp,
priv->prof->rx_pause,
priv->prof->rx_ppp);
if (err)
en_err(priv, "Failed setting pause params\n");
return err;
}
static int mlx4_en_test_loopback_xmit(struct mlx4_en_priv *priv)
{
struct sk_buff *skb;
struct ethhdr *ethh;
unsigned char *packet;
unsigned int packet_size = MLX4_LOOPBACK_TEST_PAYLOAD;
unsigned int i;
int err;
/* build the pkt before xmit */
skb = netdev_alloc_skb(priv->dev, MLX4_LOOPBACK_TEST_PAYLOAD + ETH_HLEN + NET_IP_ALIGN);
if (!skb) {
en_err(priv, "-LOOPBACK_TEST_XMIT- failed to create skb for xmit\n");
return -ENOMEM;
}
skb_reserve(skb, NET_IP_ALIGN);
ethh = (struct ethhdr *)skb_put(skb, sizeof(struct ethhdr));
packet = (unsigned char *)skb_put(skb, packet_size);
memcpy(ethh->h_dest, priv->dev->dev_addr, ETH_ALEN);
memset(ethh->h_source, 0, ETH_ALEN);
ethh->h_proto = htons(ETH_P_ARP);
skb_set_mac_header(skb, 0);
for (i = 0; i < packet_size; ++i) /* fill our packet */
packet[i] = (unsigned char)(i & 0xff);
/* xmit the pkt */
err = mlx4_en_xmit(skb, priv->dev);
return err;
}
int mlx4_en_create_cq(struct mlx4_en_priv *priv,
struct mlx4_en_cq **pcq,
int entries, int ring, enum cq_type mode,
int node)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_cq *cq;
int err;
cq = kzalloc_node(sizeof(struct mlx4_en_cq), GFP_KERNEL, node);
if (!cq) {
cq = kzalloc(sizeof(struct mlx4_en_cq), GFP_KERNEL);
if (!cq) {
en_err(priv, "Failed to allocate CW struture\n");
return -ENOMEM;
}
}
cq->size = entries;
cq->buf_size = cq->size * mdev->dev->caps.cqe_size;
cq->tq = taskqueue_create_fast("mlx4_en_que", M_NOWAIT,
taskqueue_thread_enqueue, &cq->tq);
if (mode == RX) {
TASK_INIT(&cq->cq_task, 0, mlx4_en_rx_que, cq);
taskqueue_start_threads(&cq->tq, 1, PI_NET, "%s rx cq",
if_name(priv->dev));
} else {
TASK_INIT(&cq->cq_task, 0, mlx4_en_tx_que, cq);
taskqueue_start_threads(&cq->tq, 1, PI_NET, "%s tx cq",
if_name(priv->dev));
}
cq->ring = ring;
cq->is_tx = mode;
spin_lock_init(&cq->lock);
err = mlx4_alloc_hwq_res(mdev->dev, &cq->wqres,
cq->buf_size, 2 * PAGE_SIZE);
if (err)
goto err_cq;
err = mlx4_en_map_buffer(&cq->wqres.buf);
if (err)
goto err_res;
cq->buf = (struct mlx4_cqe *) cq->wqres.buf.direct.buf;
*pcq = cq;
return 0;
err_res:
mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size);
err_cq:
kfree(cq);
return err;
}
int mlx4_en_timestamp_config(struct net_device *dev, int tx_type, int rx_filter)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_en_dev *mdev = priv->mdev;
int port_up = 0;
int err = 0;
mutex_lock(&mdev->state_lock);
if (priv->port_up) {
port_up = 1;
mlx4_en_stop_port(dev, 1);
}
mlx4_en_free_resources(priv);
en_warn(priv, "Changing Time Stamp configuration\n");
priv->hwtstamp_config.tx_type = tx_type;
priv->hwtstamp_config.rx_filter = rx_filter;
if (rx_filter != HWTSTAMP_FILTER_NONE)
dev->features &= ~NETIF_F_HW_VLAN_RX;
else
dev->features |= NETIF_F_HW_VLAN_RX;
err = mlx4_en_alloc_resources(priv);
if (err) {
en_err(priv, "Failed reallocating port resources\n");
goto out;
}
if (port_up) {
err = mlx4_en_start_port(dev);
if (err)
en_err(priv, "Failed starting port\n");
}
out:
mutex_unlock(&mdev->state_lock);
netdev_features_change(dev);
return err;
}
static int mlx4_en_test_loopback(struct mlx4_en_priv *priv)
{
uint32_t loopback_ok = 0;
int i;
bool gro_enabled;
priv->loopback_ok = 0;
priv->validate_loopback = 1;
gro_enabled = priv->dev->feat & NETIF_F_GRO;
mlx4_en_update_loopback_state(priv->dev, priv->dev->feat);
priv->dev->feat &= ~NETIF_F_GRO;
/* xmit */
if (mlx4_en_test_loopback_xmit(priv)) {
en_err(priv, "Transmitting loopback packet failed\n");
goto mlx4_en_test_loopback_exit;
}
/* polling for result */
for (i = 0; i < MLX4_EN_LOOPBACK_RETRIES; ++i) {
kthread_usleep(1000 * MLX4_EN_LOOPBACK_TIMEOUT);
if (priv->loopback_ok) {
loopback_ok = 1;
break;
}
}
if (!loopback_ok)
en_err(priv, "Loopback packet didn't arrive\n");
mlx4_en_test_loopback_exit:
priv->validate_loopback = 0;
if (gro_enabled)
priv->dev->feat |= NETIF_F_GRO;
mlx4_en_update_loopback_state(priv->dev, priv->dev->feat);
return !loopback_ok;
}
int mlx4_en_create_cq(struct mlx4_en_priv *priv,
struct mlx4_en_cq *cq,
int entries, int ring, enum cq_type mode)
{
struct mlx4_en_dev *mdev = priv->mdev;
int err;
cq->size = entries;
if (mode == RX) {
cq->buf_size = cq->size * sizeof(struct mlx4_cqe);
cq->vector = (ring + priv->port) %
mdev->dev->caps.num_comp_vectors;
} else {
cq->buf_size = sizeof(struct mlx4_cqe);
cq->vector = 0;
}
cq->ring = ring;
cq->is_tx = mode;
if (priv->rx_ring[ring].use_frags)
cq->process_cq = mlx4_en_process_rx_cq;
else
cq->process_cq = mlx4_en_process_rx_cq_skb;
spin_lock_init(&cq->lock);
err = mlx4_alloc_hwq_res(mdev->dev, &cq->wqres,
cq->buf_size, 2 * PAGE_SIZE);
if (err) {
en_err(priv, "Failed to allocate CQ buffer\n");
return err;
}
err = mlx4_en_map_buffer(&cq->wqres.buf);
if (err) {
en_err(priv, "Failed to map CQ buffer\n");
mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size);
} else
cq->buf = (struct mlx4_cqe *) cq->wqres.buf.direct.buf;
return err;
}
int mlx4_en_create_cq(struct mlx4_en_priv *priv,
struct mlx4_en_cq **pcq,
int entries, int ring, enum cq_type mode,
int node)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_cq *cq;
int err;
cq = kzalloc_node(sizeof(*cq), GFP_KERNEL, node);
if (!cq) {
cq = kzalloc(sizeof(*cq), GFP_KERNEL);
if (!cq) {
en_err(priv, "Failed to allocate CQ structure\n");
return -ENOMEM;
}
}
cq->size = entries;
cq->buf_size = cq->size * mdev->dev->caps.cqe_size;
cq->ring = ring;
cq->is_tx = mode;
cq->vector = mdev->dev->caps.num_comp_vectors;
/* Allocate HW buffers on provided NUMA node.
* dev->numa_node is used in mtt range allocation flow.
*/
set_dev_node(&mdev->dev->persist->pdev->dev, node);
err = mlx4_alloc_hwq_res(mdev->dev, &cq->wqres,
cq->buf_size, 2 * PAGE_SIZE);
set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
if (err)
goto err_cq;
err = mlx4_en_map_buffer(&cq->wqres.buf);
if (err)
goto err_res;
cq->buf = (struct mlx4_cqe *)cq->wqres.buf.direct.buf;
*pcq = cq;
return 0;
err_res:
mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size);
err_cq:
kfree(cq);
*pcq = NULL;
return err;
}
static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
struct mlx4_en_rx_ring *ring,
enum mlx4_qp_state *state,
struct mlx4_qp *qp)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_qp_context *context;
int err = 0;
context = kmalloc(sizeof *context , GFP_KERNEL);
if (!context) {
en_err(priv, "Failed to allocate qp context\n");
return -ENOMEM;
}
err = mlx4_qp_alloc(mdev->dev, qpn, qp);
if (err) {
en_err(priv, "Failed to allocate qp #%x\n", qpn);
goto out;
}
qp->event = mlx4_en_sqp_event;
memset(context, 0, sizeof *context);
mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
qpn, ring->cqn, context);
context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
if (err) {
mlx4_qp_remove(mdev->dev, qp);
mlx4_qp_free(mdev->dev, qp);
}
mlx4_en_update_rx_prod_db(ring);
out:
kfree(context);
return err;
}
static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
{
struct mlx4_en_rx_ring *ring;
int ring_ind;
int buf_ind;
int new_size;
int err;
for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = priv->rx_ring[ring_ind];
err = mlx4_en_prepare_rx_desc(priv, ring,
ring->actual_size);
if (err) {
if (ring->actual_size == 0) {
en_err(priv, "Failed to allocate "
"enough rx buffers\n");
return -ENOMEM;
} else {
new_size =
rounddown_pow_of_two(ring->actual_size);
en_warn(priv, "Only %d buffers allocated "
"reducing ring size to %d\n",
ring->actual_size, new_size);
goto reduce_rings;
}
}
ring->actual_size++;
ring->prod++;
}
}
return 0;
reduce_rings:
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = priv->rx_ring[ring_ind];
while (ring->actual_size > new_size) {
ring->actual_size--;
ring->prod--;
mlx4_en_free_buf(ring,
ring->mbuf + ring->actual_size);
}
}
return 0;
}
static void mlx4_en_destroy_rl_res(struct mlx4_en_priv *priv,
int ring_id)
{
struct mlx4_en_tx_ring *ring;
struct mlx4_en_dev *mdev = priv->mdev;
ring = priv->tx_ring[ring_id];
mutex_lock(&mdev->state_lock);
/* Index was validated, thus ring is not NULL */
spin_lock(&ring->tx_lock);
if (ring->rl_data.user_valid == false) {
en_err(priv, "ring %d doesn't exist\n", ring_id);
spin_unlock(&ring->tx_lock);
return;
} else {
ring->rl_data.user_valid = false;
}
if (!drbr_empty(priv->dev, ring->br)) {
struct mbuf *m;
while ((m = buf_ring_dequeue_sc(ring->br)) != NULL) {
m_freem(m);
}
}
spin_unlock(&ring->tx_lock);
atomic_subtract_int(&priv->rate_limits[ring->rl_data.rate_index].ref, 1);
/* Deactivate resources */
if (priv->port_up) {
mlx4_en_deactivate_tx_ring(priv, ring);
mlx4_en_deactivate_cq(priv, priv->tx_cq[ring_id]);
msleep(10);
mlx4_en_free_tx_buf(priv->dev, ring);
}
mutex_unlock(&mdev->state_lock);
/* clear statistics */
ring->bytes = 0;
ring->packets = 0;
sysctl_ctx_free(&ring->rl_data.rl_stats_ctx);
/* Add index to re-use list */
priv->rate_limit_tx_ring_num--;
mlx4_en_rl_reused_index_insert(priv, ring_id);
}
static inline int invalid_cqe(struct mlx4_en_priv *priv,
struct mlx4_cqe *cqe)
{
/* Drop packet on bad receive or bad checksum */
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
en_err(priv, "CQE completed in error - vendor "
"syndrom:%d syndrom:%d\n",
((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome,
((struct mlx4_err_cqe *) cqe)->syndrome);
return 1;
}
if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
return 1;
}
return 0;
}
static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
{
struct mlx4_en_rx_ring *ring;
int ring_ind;
int buf_ind;
int new_size;
for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = priv->rx_ring[ring_ind];
if (mlx4_en_prepare_rx_desc(priv, ring,
ring->actual_size,
GFP_KERNEL | __GFP_COLD)) {
if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
en_err(priv, "Failed to allocate enough rx buffers\n");
return -ENOMEM;
} else {
new_size = rounddown_pow_of_two(ring->actual_size);
en_warn(priv, "Only %d buffers allocated reducing ring size to %d\n",
ring->actual_size, new_size);
goto reduce_rings;
}
}
ring->actual_size++;
ring->prod++;
}
}
return 0;
reduce_rings:
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = priv->rx_ring[ring_ind];
while (ring->actual_size > new_size) {
ring->actual_size--;
ring->prod--;
mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
}
}
return 0;
}
int mlx4_en_destroy_rate_limit_ring(struct mlx4_en_priv *priv,
struct ifreq_hwtxring *rl_req)
{
uint32_t ring_id;
int err = 0;
ring_id = rl_req->txringid;
/* Check that this is indeed a rate limit ring */
if (ring_id < priv->native_tx_ring_num || ring_id >= priv->tx_ring_num) {
en_err(priv, "Deleting ring %d: Permision denied: Not a rate limit ring\n", ring_id);
return (EINVAL);
}
/* Defer ring destruction */
/* There is no handling with new rate index when destroying a ring
* therefor, sending zero as a rate index. */
err = mlx4_en_defer_rl_op(priv, rl_req->txringid, 0, MLX4_EN_RL_DEL);
return err;
}
static int mlx4_en_defer_rl_op(struct mlx4_en_priv *priv,
int ring_id, u8 rate_index,
enum mlx4_en_rl_operation opp)
{
struct mlx4_en_rl_task_list_element *rl_item;
rl_item = kmalloc(sizeof(struct mlx4_en_rl_task_list_element), M_NOWAIT);
if (!rl_item) {
en_err(priv, "Failed allocating rl_item\n");
return (ENOMEM);
}
/* Saving recieved data from kernel in order to use it later in
* the defer function */
rl_item->ring_id = ring_id;
rl_item->rate_index = rate_index;
rl_item->operation = opp;
spin_lock(&priv->rl_op_lock);
STAILQ_INSERT_TAIL(&priv->rl_op_list_head, rl_item, entry);
spin_unlock(&priv->rl_op_lock);
taskqueue_enqueue(priv->rl_tq, &priv->rl_task);
return (0);
}
int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cqe *cqe;
struct mlx4_en_rx_ring *ring = &priv->rx_ring[cq->ring];
struct skb_frag_struct *skb_frags;
struct mlx4_en_rx_desc *rx_desc;
struct sk_buff *skb;
int index;
int nr;
unsigned int length;
int polled = 0;
int ip_summed;
if (!priv->port_up)
return 0;
/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
* descriptor offset can be deduced from the CQE index instead of
* reading 'cqe->index' */
index = cq->mcq.cons_index & ring->size_mask;
cqe = &cq->buf[index];
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cq->mcq.cons_index & cq->size)) {
skb_frags = ring->rx_info + (index << priv->log_rx_info);
rx_desc = ring->buf + (index << ring->log_stride);
/*
* make sure we read the CQE after we read the ownership bit
*/
rmb();
/* Drop packet on bad receive or bad checksum */
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
en_err(priv, "CQE completed in error - vendor "
"syndrom:%d syndrom:%d\n",
((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome,
((struct mlx4_err_cqe *) cqe)->syndrome);
goto next;
}
if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
goto next;
}
/*
* Packet is OK - process it.
*/
length = be32_to_cpu(cqe->byte_cnt);
ring->bytes += length;
ring->packets++;
if (likely(dev->features & NETIF_F_RXCSUM)) {
if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
(cqe->checksum == cpu_to_be16(0xffff))) {
priv->port_stats.rx_chksum_good++;
/* This packet is eligible for LRO if it is:
* - DIX Ethernet (type interpretation)
* - TCP/IP (v4)
* - without IP options
* - not an IP fragment */
if (dev->features & NETIF_F_GRO) {
struct sk_buff *gro_skb = napi_get_frags(&cq->napi);
if (!gro_skb)
goto next;
nr = mlx4_en_complete_rx_desc(
priv, rx_desc,
skb_frags, skb_shinfo(gro_skb)->frags,
ring->page_alloc, length);
if (!nr)
goto next;
skb_shinfo(gro_skb)->nr_frags = nr;
gro_skb->len = length;
gro_skb->data_len = length;
gro_skb->truesize += length;
gro_skb->ip_summed = CHECKSUM_UNNECESSARY;
if (priv->vlgrp && (cqe->vlan_my_qpn &
cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK)))
vlan_gro_frags(&cq->napi, priv->vlgrp, be16_to_cpu(cqe->sl_vid));
else
napi_gro_frags(&cq->napi);
goto next;
}
/* LRO not possible, complete processing here */
ip_summed = CHECKSUM_UNNECESSARY;
} else {
ip_summed = CHECKSUM_NONE;
priv->port_stats.rx_chksum_none++;
}
} else {
ip_summed = CHECKSUM_NONE;
priv->port_stats.rx_chksum_none++;
}
skb = mlx4_en_rx_skb(priv, rx_desc, skb_frags,
ring->page_alloc, length);
if (!skb) {
priv->stats.rx_dropped++;
goto next;
}
if (unlikely(priv->validate_loopback)) {
validate_loopback(priv, skb);
goto next;
}
skb->ip_summed = ip_summed;
skb->protocol = eth_type_trans(skb, dev);
skb_record_rx_queue(skb, cq->ring);
/* Push it up the stack */
if (priv->vlgrp && (be32_to_cpu(cqe->vlan_my_qpn) &
MLX4_CQE_VLAN_PRESENT_MASK)) {
vlan_hwaccel_receive_skb(skb, priv->vlgrp,
be16_to_cpu(cqe->sl_vid));
} else
netif_receive_skb(skb);
next:
++cq->mcq.cons_index;
index = (cq->mcq.cons_index) & ring->size_mask;
cqe = &cq->buf[index];
if (++polled == budget) {
/* We are here because we reached the NAPI budget -
* flush only pending LRO sessions */
goto out;
}
}
out:
AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
mlx4_cq_set_ci(&cq->mcq);
wmb(); /* ensure HW sees CQ consumer before we post new buffers */
ring->cons = cq->mcq.cons_index;
ring->prod += polled; /* Polled descriptors were realocated in place */
mlx4_en_update_rx_prod_db(ring);
return polled;
}
static int mlx4_en_process_tx_cq(struct net_device *dev,
struct mlx4_en_cq *cq)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring];
struct mlx4_cqe *cqe;
u16 index;
u16 new_index, ring_index, stamp_index;
u32 txbbs_skipped = 0;
#ifndef CONFIG_WQE_FORMAT_1
u32 txbbs_stamp = 0;
#endif
u32 cons_index = mcq->cons_index;
int size = cq->size;
u32 size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
u32 packets = 0;
u32 bytes = 0;
int factor = priv->cqe_factor;
u64 timestamp = 0;
int done = 0;
if (!priv->port_up)
return 0;
index = cons_index & size_mask;
cqe = &buf[(index << factor) + factor];
ring_index = ring->cons & size_mask;
stamp_index = ring_index;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size)) {
/*
* make sure we read the CQE after we read the
* ownership bit
*/
rmb();
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
en_err(priv, "CQE completed in error - vendor syndrom: 0x%x syndrom: 0x%x\n",
((struct mlx4_err_cqe *)cqe)->
vendor_err_syndrome,
((struct mlx4_err_cqe *)cqe)->syndrome);
}
/* Skip over last polled CQE */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
txbbs_skipped += ring->last_nr_txbb;
ring_index = (ring_index + ring->last_nr_txbb) & size_mask;
/* free next descriptor */
ring->last_nr_txbb = mlx4_en_free_tx_desc(
priv, ring, ring_index,
!!((ring->cons + txbbs_skipped) &
ring->size), timestamp);
#ifndef CONFIG_WQE_FORMAT_1
mlx4_en_stamp_wqe(priv, ring, stamp_index,
!!((ring->cons + txbbs_stamp) &
ring->size));
stamp_index = ring_index;
txbbs_stamp = txbbs_skipped;
#endif
packets++;
bytes += ring->tx_info[ring_index].nr_bytes;
} while (ring_index != new_index);
++cons_index;
index = cons_index & size_mask;
cqe = &buf[(index << factor) + factor];
}
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
ring->cons += txbbs_skipped;
/* Wakeup Tx queue if it was stopped and ring is not full */
if (unlikely(ring->blocked) &&
(ring->prod - ring->cons) <= ring->full_size) {
ring->blocked = 0;
#ifdef CONFIG_RATELIMIT
if (cq->ring < priv->native_tx_ring_num) {
if (atomic_fetchadd_int(&priv->blocked, -1) == 1)
atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE);
priv->port_stats.wake_queue++;
}
#else
if (atomic_fetchadd_int(&priv->blocked, -1) == 1)
atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE);
priv->port_stats.wake_queue++;
#endif
ring->wake_queue++;
}
return done;
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring, u32 size,
u16 stride, int node, int queue_idx)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring;
int tmp;
int err;
ring = kzalloc_node(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL, node);
if (!ring) {
ring = kzalloc(sizeof(struct mlx4_en_tx_ring), GFP_KERNEL);
if (!ring) {
en_err(priv, "Failed allocating TX ring\n");
return -ENOMEM;
}
}
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
#ifdef CONFIG_RATELIMIT
ring->rl_data.rate_index = 0;
/* User_valid should be false in a rate_limit ring until the
* creation process of the ring is done, after the activation. */
if (queue_idx < priv->native_tx_ring_num)
ring->rl_data.user_valid = true;
else
ring->rl_data.user_valid = false;
#endif
ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS;
ring->inline_thold = min(inline_thold, MAX_INLINE);
mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF);
mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF);
/* Allocate the buf ring */
#ifdef CONFIG_RATELIMIT
if (queue_idx < priv->native_tx_ring_num)
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
M_WAITOK, &ring->tx_lock.m);
else
ring->br = buf_ring_alloc(size / 4, M_DEVBUF, M_WAITOK,
&ring->tx_lock.m);
#else
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
M_WAITOK, &ring->tx_lock.m);
#endif
if (ring->br == NULL) {
en_err(priv, "Failed allocating tx_info ring\n");
return -ENOMEM;
}
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = vmalloc_node(tmp, node);
if (!ring->tx_info) {
ring->tx_info = vmalloc(tmp);
if (!ring->tx_info) {
err = -ENOMEM;
goto err_ring;
}
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node);
if (!ring->bounce_buf) {
ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
if (!ring->bounce_buf) {
err = -ENOMEM;
goto err_info;
}
}
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
/* Allocate HW buffers on provided NUMA node */
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_bounce;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d "
"buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_BF_QP);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_map;
}
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_reserve;
}
ring->qp.event = mlx4_en_sqp_event;
err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
if (err) {
en_dbg(DRV, priv, "working without blueflame (%d)", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
} else
ring->bf_enabled = true;
ring->queue_index = queue_idx;
if (queue_idx < priv->num_tx_rings_p_up )
CPU_SET(queue_idx, &ring->affinity_mask);
*pring = ring;
return 0;
err_reserve:
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_bounce:
kfree(ring->bounce_buf);
err_info:
vfree(ring->tx_info);
err_ring:
buf_ring_free(ring->br, M_DEVBUF);
kfree(ring);
return err;
}
int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring, u32 size, u16 stride)
{
struct mlx4_en_dev *mdev = priv->mdev;
int err;
int tmp;
ring->prod = 0;
ring->cons = 0;
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
ring->log_stride = ffs(ring->stride) - 1;
ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
sizeof(struct skb_frag_struct));
ring->rx_info = vmalloc(tmp);
if (!ring->rx_info) {
en_err(priv, "Failed allocating rx_info ring\n");
return -ENOMEM;
}
en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
ring->rx_info, tmp);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
ring->buf_size, 2 * PAGE_SIZE);
if (err)
goto err_ring;
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map RX buffer\n");
goto err_hwq;
}
ring->buf = ring->wqres.buf.direct.buf;
/* Configure lro mngr */
memset(&ring->lro, 0, sizeof(struct net_lro_mgr));
ring->lro.dev = priv->dev;
ring->lro.features = LRO_F_NAPI;
ring->lro.frag_align_pad = NET_IP_ALIGN;
ring->lro.ip_summed = CHECKSUM_UNNECESSARY;
ring->lro.ip_summed_aggr = CHECKSUM_UNNECESSARY;
ring->lro.max_desc = mdev->profile.num_lro;
ring->lro.max_aggr = MAX_SKB_FRAGS;
ring->lro.lro_arr = kzalloc(mdev->profile.num_lro *
sizeof(struct net_lro_desc),
GFP_KERNEL);
if (!ring->lro.lro_arr) {
en_err(priv, "Failed to allocate lro array\n");
goto err_map;
}
ring->lro.get_frag_header = mlx4_en_get_frag_header;
return 0;
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_ring:
vfree(ring->rx_info);
ring->rx_info = NULL;
return err;
}
static void mlx4_en_create_rl_res(struct mlx4_en_priv *priv,
int ring_id, u8 rate_index)
{
struct mlx4_en_cq *cq;
struct mlx4_en_tx_ring *tx_ring;
struct mlx4_en_dev *mdev = priv->mdev;
int err = 0;
int node = 0;
int j;
if (priv->tx_ring[ring_id]) {
/* Ring already exists, needs activation */
/* Make sure drbr queue has no left overs from before */
tx_ring = priv->tx_ring[ring_id];
goto activate;
}
err = mlx4_en_create_cq(priv, &priv->tx_cq[ring_id],
MLX4_EN_DEF_RL_TX_RING_SIZE, ring_id, TX, node);
if (err) {
en_err(priv, "Failed to create rate limit tx CQ, ring index %u, rate %u\n", ring_id, rate_index);
goto err_create_cq;
}
err = mlx4_en_create_tx_ring(priv, &priv->tx_ring[ring_id],
MLX4_EN_DEF_RL_TX_RING_SIZE, TXBB_SIZE, node, ring_id);
if (err) {
en_err(priv, "Failed to create rate limited tx ring %u, rate %u\n", ring_id, rate_index);
goto err_create_ring;
}
tx_ring = priv->tx_ring[ring_id];
activate:
sysctl_ctx_init(&tx_ring->rl_data.rl_stats_ctx);
tx_ring->rl_data.rate_index = rate_index;
/* Default moderation */
cq = priv->tx_cq[ring_id];
cq->moder_cnt = priv->tx_frames;
cq->moder_time = priv->tx_usecs;
mutex_lock(&mdev->state_lock);
if (!priv->port_up) {
/* No need activating resources, start_port will take care of that */
tx_ring->rl_data.user_valid = true;
mutex_unlock(&mdev->state_lock);
return;
}
/* Activate resources */
err = mlx4_en_activate_cq(priv, cq, ring_id);
if (err) {
en_err(priv, "Failed activating Rate Limit Tx CQ\n");
goto err_activate_resources;
}
err = mlx4_en_set_cq_moder(priv, cq);
if (err) {
en_err(priv, "Failed setting cq moderation parameters");
mlx4_en_deactivate_cq(priv, cq);
goto err_activate_resources;
}
en_dbg(DRV, priv, "Resetting index of CQ:%d to -1\n", ring_id);
cq->buf->wqe_index = cpu_to_be16(0xffff);
err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn,
MLX4_EN_DEF_RL_USER_PRIO);
if (err) {
en_err(priv, "Failed activating rate limit TX ring\n");
mlx4_en_deactivate_cq(priv, cq);
goto err_activate_resources;
}
/* Arm CQ for TX completions */
mlx4_en_arm_cq(priv, cq);
/* Set initial ownership of all Tx TXBBs to SW (1) */
for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE)
*((u32 *) (tx_ring->buf + j)) = INIT_OWNER_BIT;
/* Set ring as valid */
tx_ring->rl_data.user_valid = true;
mutex_unlock(&mdev->state_lock);
priv->rate_limit_tx_ring_num++;
/* Add rate limit statistics to sysctl if debug option was enabled */
if (show_rl_sysctl_info)
mlx4_en_rate_limit_sysctl_stat(priv, ring_id);
return;
err_activate_resources:
mlx4_en_invalidate_rl_ring(priv, ring_id);
mlx4_en_rl_reused_index_insert(priv, ring_id);
atomic_subtract_int(&priv->rate_limits[rate_index].ref, 1);
mutex_unlock(&mdev->state_lock);
return;
err_create_ring:
if (priv->tx_cq[ring_id])
mlx4_en_destroy_cq(priv, &priv->tx_cq[ring_id]);
err_create_cq:
mlx4_en_rl_reused_index_insert(priv, ring_id);
atomic_subtract_int(&priv->rate_limits[rate_index].ref, 1);
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring, uint32_t size,
uint16_t stride, int node, int queue_index)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring;
int tmp;
int err;
ring = kzalloc_node(sizeof(*ring), KMALLOC_WAIT, node);
if (!ring) {
ring = kzmalloc(sizeof(*ring), KMALLOC_WAIT);
if (!ring) {
en_err(priv, "Failed allocating TX ring\n");
return -ENOMEM;
}
}
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = kmalloc_node(tmp, KMALLOC_WAIT | __GFP_NOWARN, node);
if (!ring->tx_info) {
ring->tx_info = vmalloc(tmp);
if (!ring->tx_info) {
err = -ENOMEM;
goto err_ring;
}
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, KMALLOC_WAIT, node);
if (!ring->bounce_buf) {
ring->bounce_buf = kmalloc(MAX_DESC_SIZE, KMALLOC_WAIT);
if (!ring->bounce_buf) {
err = -ENOMEM;
goto err_info;
}
}
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
/* Allocate HW buffers on provided NUMA node */
set_dev_node(&mdev->dev->persist->pdev->dev, node);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_bounce;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
ring, ring->buf, ring->size, ring->buf_size,
(unsigned long long) ring->wqres.buf.direct.map);
err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_ETH_BF_QP);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_map;
}
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp, KMALLOC_WAIT);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_reserve;
}
ring->qp.event = mlx4_en_sqp_event;
#if 0 // AKAROS_PORT
err = mlx4_bf_alloc(mdev->dev, &ring->bf, node);
if (err) {
#else
if (true) {
#endif
en_dbg(DRV, priv, "working without blueflame (%d)\n", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
ring->bf_alloced = false;
priv->pflags &= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME;
} else {
ring->bf_alloced = true;
ring->bf_enabled = !!(priv->pflags &
MLX4_EN_PRIV_FLAGS_BLUEFLAME);
}
ring->hwtstamp_tx_type = priv->hwtstamp_config.tx_type;
ring->queue_index = queue_index;
if (queue_index < priv->num_tx_rings_p_up)
cpumask_set_cpu(cpumask_local_spread(queue_index,
priv->mdev->dev->numa_node),
&ring->affinity_mask);
*pring = ring;
return 0;
err_reserve:
mlx4_qp_release_range(mdev->dev, ring->qpn, 1);
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_bounce:
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
err_info:
#if 0 // AKAROS_PORT
kvfree(ring->tx_info);
#endif
ring->tx_info = NULL;
err_ring:
kfree(ring);
*pring = NULL;
return err;
}
void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring **pring)
{
panic("Disabled");
#if 0 // AKAROS_PORT
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_tx_ring *ring = *pring;
en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn);
if (ring->bf_alloced)
mlx4_bf_free(mdev->dev, &ring->bf);
mlx4_qp_remove(mdev->dev, &ring->qp);
mlx4_qp_free(mdev->dev, &ring->qp);
mlx4_en_unmap_buffer(&ring->wqres.buf);
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
kvfree(ring->tx_info);
ring->tx_info = NULL;
kfree(ring);
*pring = NULL;
#endif
}
static bool mlx4_en_process_tx_cq(struct ether *dev,
struct mlx4_en_cq *cq)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_cq *mcq = &cq->mcq;
struct mlx4_en_tx_ring *ring = priv->tx_ring[cq->ring];
struct mlx4_cqe *cqe;
uint16_t index;
uint16_t new_index, ring_index, stamp_index;
uint32_t txbbs_skipped = 0;
uint32_t txbbs_stamp = 0;
uint32_t cons_index = mcq->cons_index;
int size = cq->size;
uint32_t size_mask = ring->size_mask;
struct mlx4_cqe *buf = cq->buf;
uint32_t packets = 0;
uint32_t bytes = 0;
int factor = priv->cqe_factor;
uint64_t timestamp = 0;
int done = 0;
int budget = priv->tx_work_limit;
uint32_t last_nr_txbb;
uint32_t ring_cons;
if (!priv->port_up)
return true;
#if 0 // AKAROS_PORT
netdev_txq_bql_complete_prefetchw(ring->tx_queue);
#endif
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
last_nr_txbb = ACCESS_ONCE(ring->last_nr_txbb);
ring_cons = ACCESS_ONCE(ring->cons);
ring_index = ring_cons & size_mask;
stamp_index = ring_index;
/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size) && (done < budget)) {
/*
* make sure we read the CQE after we read the
* ownership bit
*/
bus_rmb();
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
MLX4_CQE_OPCODE_ERROR)) {
struct mlx4_err_cqe *cqe_err = (struct mlx4_err_cqe *)cqe;
en_err(priv, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
cqe_err->vendor_err_syndrome,
cqe_err->syndrome);
}
/* Skip over last polled CQE */
new_index = be16_to_cpu(cqe->wqe_index) & size_mask;
do {
txbbs_skipped += last_nr_txbb;
ring_index = (ring_index + last_nr_txbb) & size_mask;
if (ring->tx_info[ring_index].ts_requested)
timestamp = mlx4_en_get_cqe_ts(cqe);
/* free next descriptor */
last_nr_txbb = mlx4_en_free_tx_desc(
priv, ring, ring_index,
!!((ring_cons + txbbs_skipped) &
ring->size), timestamp);
mlx4_en_stamp_wqe(priv, ring, stamp_index,
!!((ring_cons + txbbs_stamp) &
ring->size));
stamp_index = ring_index;
txbbs_stamp = txbbs_skipped;
packets++;
bytes += ring->tx_info[ring_index].nr_bytes;
} while ((++done < budget) && (ring_index != new_index));
++cons_index;
index = cons_index & size_mask;
cqe = mlx4_en_get_cqe(buf, index, priv->cqe_size) + factor;
}
/*
* To prevent CQ overflow we first update CQ consumer and only then
* the ring consumer.
*/
mcq->cons_index = cons_index;
mlx4_cq_set_ci(mcq);
wmb();
/* we want to dirty this cache line once */
ACCESS_ONCE(ring->last_nr_txbb) = last_nr_txbb;
ACCESS_ONCE(ring->cons) = ring_cons + txbbs_skipped;
#if 0 // AKAROS_PORT
netdev_tx_completed_queue(ring->tx_queue, packets, bytes);
/*
* Wakeup Tx queue if this stopped, and at least 1 packet
* was completed
*/
if (netif_tx_queue_stopped(ring->tx_queue) && txbbs_skipped > 0) {
netif_tx_wake_queue(ring->tx_queue);
ring->wake_queue++;
}
#endif
return done < budget;
}
int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_tx_ring *ring, int qpn, u32 size,
u16 stride)
{
struct mlx4_en_dev *mdev = priv->mdev;
int tmp;
int err;
ring->size = size;
ring->size_mask = size - 1;
ring->stride = stride;
inline_thold = min(inline_thold, MAX_INLINE);
spin_lock_init(&ring->comp_lock);
tmp = size * sizeof(struct mlx4_en_tx_info);
ring->tx_info = vmalloc(tmp);
if (!ring->tx_info) {
en_err(priv, "Failed allocating tx_info ring\n");
return -ENOMEM;
}
en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n",
ring->tx_info, tmp);
ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL);
if (!ring->bounce_buf) {
en_err(priv, "Failed allocating bounce buffer\n");
err = -ENOMEM;
goto err_tx;
}
ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE);
err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size,
2 * PAGE_SIZE);
if (err) {
en_err(priv, "Failed allocating hwq resources\n");
goto err_bounce;
}
err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {
en_err(priv, "Failed to map TX buffer\n");
goto err_hwq_res;
}
ring->buf = ring->wqres.buf.direct.buf;
en_dbg(DRV, priv, "Allocated TX ring (addr:%p) - buf:%p size:%d "
"buf_size:%d dma:%llx\n", ring, ring->buf, ring->size,
ring->buf_size, (unsigned long long) ring->wqres.buf.direct.map);
ring->qpn = qpn;
err = mlx4_qp_alloc(mdev->dev, ring->qpn, &ring->qp);
if (err) {
en_err(priv, "Failed allocating qp %d\n", ring->qpn);
goto err_map;
}
ring->qp.event = mlx4_en_sqp_event;
err = mlx4_bf_alloc(mdev->dev, &ring->bf);
if (err) {
en_dbg(DRV, priv, "working without blueflame (%d)", err);
ring->bf.uar = &mdev->priv_uar;
ring->bf.uar->map = mdev->uar_map;
ring->bf_enabled = false;
} else
ring->bf_enabled = true;
return 0;
err_map:
mlx4_en_unmap_buffer(&ring->wqres.buf);
err_hwq_res:
mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
err_bounce:
kfree(ring->bounce_buf);
ring->bounce_buf = NULL;
err_tx:
vfree(ring->tx_info);
ring->tx_info = NULL;
return err;
}
/* Allocate rx qp's and configure them according to rss map */
int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_rss_map *rss_map = &priv->rss_map;
struct mlx4_qp_context context;
struct mlx4_en_rss_context *rss_context;
void *ptr;
u8 rss_mask = 0x3f;
int i, qpn;
int err = 0;
int good_qps = 0;
en_dbg(DRV, priv, "Configuring rss steering\n");
err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
priv->rx_ring_num,
&rss_map->base_qpn);
if (err) {
en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
return err;
}
for (i = 0; i < priv->rx_ring_num; i++) {
qpn = rss_map->base_qpn + i;
err = mlx4_en_config_rss_qp(priv, qpn, &priv->rx_ring[i],
&rss_map->state[i],
&rss_map->qps[i]);
if (err)
goto rss_err;
++good_qps;
}
/* Configure RSS indirection qp */
err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp);
if (err) {
en_err(priv, "Failed to allocate RSS indirection QP\n");
goto rss_err;
}
rss_map->indir_qp.event = mlx4_en_sqp_event;
mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
priv->rx_ring[0].cqn, &context);
ptr = ((void *) &context) + 0x3c;
rss_context = (struct mlx4_en_rss_context *) ptr;
rss_context->base_qpn = cpu_to_be32(ilog2(priv->rx_ring_num) << 24 |
(rss_map->base_qpn));
rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
rss_context->flags = rss_mask;
if (priv->mdev->profile.udp_rss)
rss_context->base_qpn_udp = rss_context->default_qpn;
err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
&rss_map->indir_qp, &rss_map->indir_state);
if (err)
goto indir_err;
return 0;
indir_err:
mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
rss_err:
for (i = 0; i < good_qps; i++) {
mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
}
mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
return err;
}
int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
{
struct mlx4_en_rx_ring *ring;
int i;
int ring_ind;
int err;
int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
DS_SIZE * priv->num_frags);
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = &priv->rx_ring[ring_ind];
ring->prod = 0;
ring->cons = 0;
ring->actual_size = 0;
ring->cqn = priv->rx_cq[ring_ind].mcq.cqn;
ring->stride = stride;
if (ring->stride <= TXBB_SIZE)
ring->buf += TXBB_SIZE;
ring->log_stride = ffs(ring->stride) - 1;
ring->buf_size = ring->size * ring->stride;
memset(ring->buf, 0, ring->buf_size);
mlx4_en_update_rx_prod_db(ring);
/* Initailize all descriptors */
for (i = 0; i < ring->size; i++)
mlx4_en_init_rx_desc(priv, ring, i);
/* Initialize page allocators */
err = mlx4_en_init_allocator(priv, ring);
if (err) {
en_err(priv, "Failed initializing ring allocator\n");
if (ring->stride <= TXBB_SIZE)
ring->buf -= TXBB_SIZE;
ring_ind--;
goto err_allocator;
}
}
err = mlx4_en_fill_rx_buffers(priv);
if (err)
goto err_buffers;
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = &priv->rx_ring[ring_ind];
ring->size_mask = ring->actual_size - 1;
mlx4_en_update_rx_prod_db(ring);
}
return 0;
err_buffers:
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]);
ring_ind = priv->rx_ring_num - 1;
err_allocator:
while (ring_ind >= 0) {
if (priv->rx_ring[ring_ind].stride <= TXBB_SIZE)
priv->rx_ring[ring_ind].buf -= TXBB_SIZE;
mlx4_en_destroy_allocator(priv, &priv->rx_ring[ring_ind]);
ring_ind--;
}
return err;
}
