static struct Qdisc *tbf_create_dflt_qdisc(struct Qdisc *sch, u32 limit)
{
struct Qdisc *q;
struct rtattr *rta;
int ret;
q = qdisc_create_dflt(sch->dev, &bfifo_qdisc_ops,
TC_H_MAKE(sch->handle, 1));
if (q) {
rta = kmalloc(RTA_LENGTH(sizeof(struct tc_fifo_qopt)), GFP_KERNEL);
if (rta) {
rta->rta_type = RTM_NEWQDISC;
rta->rta_len = RTA_LENGTH(sizeof(struct tc_fifo_qopt));
((struct tc_fifo_qopt *)RTA_DATA(rta))->limit = limit;
ret = q->ops->change(q, rta);
kfree(rta);
if (ret == 0)
return q;
}
qdisc_destroy(q);
}
return NULL;
}
void dev_activate(struct net_device *dev)
{
/* No queueing discipline is attached to device;
create default one i.e. pfifo_fast for devices,
which need queueing and noqueue_qdisc for
virtual interfaces
*/
if (dev->qdisc_sleeping == &noop_qdisc) {
struct Qdisc *qdisc;
if (dev->tx_queue_len) {
qdisc = qdisc_create_dflt(dev, &pfifo_fast_ops);
if (qdisc == NULL) {
printk(KERN_INFO "%s: activation failed\n", dev->name);
return;
}
write_lock(&qdisc_tree_lock);
qdisc->next = dev->qdisc_list;
dev->qdisc_list = qdisc;
write_unlock(&qdisc_tree_lock);
} else {
qdisc = &noqueue_qdisc;
}
write_lock(&qdisc_tree_lock);
dev->qdisc_sleeping = qdisc;
write_unlock(&qdisc_tree_lock);
}
spin_lock_bh(&dev->queue_lock);
if ((dev->qdisc = dev->qdisc_sleeping) != &noqueue_qdisc) {
dev->trans_start = jiffies;
dev_watchdog_up(dev);
}
spin_unlock_bh(&dev->queue_lock);
}
static int netem_init(struct Qdisc *sch, struct rtattr *opt)
{
struct netem_sched_data *q = qdisc_priv(sch);
int ret;
if (!opt)
return -EINVAL;
skb_queue_head_init(&q->delayed);
init_timer(&q->timer);
q->timer.function = netem_watchdog;
q->timer.data = (unsigned long) sch;
q->counter = 0;
q->qdisc = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops);
if (!q->qdisc) {
pr_debug("netem: qdisc create failed\n");
return -ENOMEM;
}
ret = netem_change(sch, opt);
if (ret) {
pr_debug("netem: change failed\n");
qdisc_destroy(q->qdisc);
}
return ret;
}
static int mq_init(struct Qdisc *sch, struct nlattr *opt)
{
struct net_device *dev = qdisc_dev(sch);
struct mq_sched *priv = qdisc_priv(sch);
struct netdev_queue *dev_queue;
struct Qdisc *qdisc;
unsigned int ntx;
if (sch->parent != TC_H_ROOT)
return -EOPNOTSUPP;
if (!netif_is_multiqueue(dev))
return -EOPNOTSUPP;
/* pre-allocate qdiscs, attachment can't fail */
priv->qdiscs = kcalloc(dev->num_tx_queues, sizeof(priv->qdiscs[0]),
GFP_KERNEL);
if (!priv->qdiscs)
return -ENOMEM;
for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
dev_queue = netdev_get_tx_queue(dev, ntx);
qdisc = qdisc_create_dflt(dev_queue, get_default_qdisc_ops(dev, ntx),
TC_H_MAKE(TC_H_MAJ(sch->handle),
TC_H_MIN(ntx + 1)));
if (!qdisc)
return -ENOMEM;
priv->qdiscs[ntx] = qdisc;
qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
}
sch->flags |= TCQ_F_MQROOT;
return 0;
}
void dev_activate(struct device *dev)
{
/* No queueing discipline is attached to device;
create default one i.e. pfifo_fast for devices,
which need queueing and noqueue_qdisc for
virtual interfaces
*/
if (dev->qdisc_sleeping == &noop_qdisc) {
if (dev->tx_queue_len) {
struct Qdisc *qdisc;
qdisc = qdisc_create_dflt(dev, &pfifo_fast_ops);
if (qdisc == NULL) {
printk(KERN_INFO "%s: activation failed\n", dev->name);
return;
}
dev->qdisc_sleeping = qdisc;
} else
dev->qdisc_sleeping = &noqueue_qdisc;
}
start_bh_atomic();
if ((dev->qdisc = dev->qdisc_sleeping) != &noqueue_qdisc) {
dev->qdisc->tx_timeo = 5*HZ;
dev->qdisc->tx_last = jiffies - dev->qdisc->tx_timeo;
if (!del_timer(&dev_watchdog))
dev_watchdog.expires = jiffies + 5*HZ;
add_timer(&dev_watchdog);
}
end_bh_atomic();
}
static int multiq_tune(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
struct multiq_sched_data *q = qdisc_priv(sch);
struct tc_multiq_qopt *qopt;
int i;
if (!netif_is_multiqueue(qdisc_dev(sch)))
return -EOPNOTSUPP;
if (nla_len(opt) < sizeof(*qopt))
return -EINVAL;
qopt = nla_data(opt);
qopt->bands = qdisc_dev(sch)->real_num_tx_queues;
sch_tree_lock(sch);
q->bands = qopt->bands;
for (i = q->bands; i < q->max_bands; i++) {
if (q->queues[i] != &noop_qdisc) {
struct Qdisc *child = q->queues[i];
q->queues[i] = &noop_qdisc;
qdisc_tree_reduce_backlog(child, child->q.qlen,
child->qstats.backlog);
qdisc_put(child);
}
}
sch_tree_unlock(sch);
for (i = 0; i < q->bands; i++) {
if (q->queues[i] == &noop_qdisc) {
struct Qdisc *child, *old;
child = qdisc_create_dflt(sch->dev_queue,
&pfifo_qdisc_ops,
TC_H_MAKE(sch->handle,
i + 1), extack);
if (child) {
sch_tree_lock(sch);
old = q->queues[i];
q->queues[i] = child;
if (child != &noop_qdisc)
qdisc_hash_add(child, true);
if (old != &noop_qdisc) {
qdisc_tree_reduce_backlog(old,
old->q.qlen,
old->qstats.backlog);
qdisc_put(old);
}
sch_tree_unlock(sch);
}
}
}
return 0;
}
static int prio_tune(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
struct prio_sched_data *q = qdisc_priv(sch);
struct Qdisc *queues[TCQ_PRIO_BANDS];
int oldbands = q->bands, i;
struct tc_prio_qopt *qopt;
if (nla_len(opt) < sizeof(*qopt))
return -EINVAL;
qopt = nla_data(opt);
if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < 2)
return -EINVAL;
for (i = 0; i <= TC_PRIO_MAX; i++) {
if (qopt->priomap[i] >= qopt->bands)
return -EINVAL;
}
/* Before commit, make sure we can allocate all new qdiscs */
for (i = oldbands; i < qopt->bands; i++) {
queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
TC_H_MAKE(sch->handle, i + 1),
extack);
if (!queues[i]) {
while (i > oldbands)
qdisc_put(queues[--i]);
return -ENOMEM;
}
}
prio_offload(sch, qopt);
sch_tree_lock(sch);
q->bands = qopt->bands;
memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);
for (i = q->bands; i < oldbands; i++) {
struct Qdisc *child = q->queues[i];
qdisc_tree_reduce_backlog(child, child->q.qlen,
child->qstats.backlog);
}
for (i = oldbands; i < q->bands; i++) {
q->queues[i] = queues[i];
if (q->queues[i] != &noop_qdisc)
qdisc_hash_add(q->queues[i], true);
}
sch_tree_unlock(sch);
for (i = q->bands; i < oldbands; i++)
qdisc_put(q->queues[i]);
return 0;
}
static int prio_tune(struct Qdisc *sch, struct nlattr *opt)
{
struct prio_sched_data *q = qdisc_priv(sch);
struct tc_prio_qopt *qopt;
int i;
if (nla_len(opt) < sizeof(*qopt))
return -EINVAL;
qopt = nla_data(opt);
if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < 2)
return -EINVAL;
for (i = 0; i <= TC_PRIO_MAX; i++) {
if (qopt->priomap[i] >= qopt->bands)
return -EINVAL;
}
sch_tree_lock(sch);
q->bands = qopt->bands;
memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);
for (i = q->bands; i < TCQ_PRIO_BANDS; i++) {
struct Qdisc *child = q->queues[i];
q->queues[i] = &noop_qdisc;
if (child != &noop_qdisc) {
qdisc_tree_decrease_qlen(child, child->q.qlen);
qdisc_destroy(child);
}
}
sch_tree_unlock(sch);
for (i = 0; i < q->bands; i++) {
if (q->queues[i] == &noop_qdisc) {
struct Qdisc *child, *old;
child = qdisc_create_dflt(sch->dev_queue,
&pfifo_qdisc_ops,
TC_H_MAKE(sch->handle, i + 1));
if (child) {
sch_tree_lock(sch);
old = q->queues[i];
q->queues[i] = child;
if (old != &noop_qdisc) {
qdisc_tree_decrease_qlen(old,
old->q.qlen);
qdisc_destroy(old);
}
sch_tree_unlock(sch);
}
}
}
return 0;
}
static int multiq_tune(struct Qdisc *sch, struct nlattr *opt)
{
struct multiq_sched_data *q = qdisc_priv(sch);
struct tc_multiq_qopt *qopt;
int i;
if (!netif_is_multiqueue(qdisc_dev(sch)))
return -EOPNOTSUPP;
if (nla_len(opt) < sizeof(*qopt))
return -EINVAL;
qopt = nla_data(opt);
qopt->bands = qdisc_dev(sch)->real_num_tx_queues;
sch_tree_lock(sch);
q->bands = qopt->bands;
for (i = q->bands; i < q->max_bands; i++) {
if (q->queues[i] != &noop_qdisc) {
struct Qdisc *child = q->queues[i];
q->queues[i] = &noop_qdisc;
qdisc_tree_decrease_qlen(child, child->q.qlen);
qdisc_destroy(child);
}
}
sch_tree_unlock(sch);
for (i = 0; i < q->bands; i++) {
if (q->queues[i] == &noop_qdisc) {
struct Qdisc *child, *old;
child = qdisc_create_dflt(qdisc_dev(sch),
sch->dev_queue,
&pfifo_qdisc_ops,
TC_H_MAKE(sch->handle,
i + 1));
if (child) {
sch_tree_lock(sch);
old = q->queues[i];
q->queues[i] = child;
if (old != &noop_qdisc) {
qdisc_tree_decrease_qlen(old,
old->q.qlen);
qdisc_destroy(old);
}
sch_tree_unlock(sch);
}
}
}
return 0;
}
static void attach_one_default_qdisc(struct net_device *dev,
struct netdev_queue *dev_queue,
void *_unused)
{
struct Qdisc *qdisc;
if (dev->tx_queue_len) {
qdisc = qdisc_create_dflt(dev, dev_queue,
&pfifo_fast_ops, TC_H_ROOT);
if (!qdisc) {
printk(KERN_INFO "%s: activation failed\n", dev->name);
return;
}
} else {
qdisc = &noqueue_qdisc;
}
dev_queue->qdisc_sleeping = qdisc;
}
static int prio_tune(struct Qdisc *sch, struct rtattr *opt)
{
struct prio_sched_data *q = (struct prio_sched_data *)sch->data;
struct tc_prio_qopt *qopt = RTA_DATA(opt);
int i;
if (opt->rta_len < RTA_LENGTH(sizeof(*qopt)))
return -EINVAL;
if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < 2)
return -EINVAL;
for (i=0; i<=TC_PRIO_MAX; i++) {
if (qopt->priomap[i] >= qopt->bands)
return -EINVAL;
}
sch_tree_lock(sch);
q->bands = qopt->bands;
memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);
for (i=q->bands; i<TCQ_PRIO_BANDS; i++) {
struct Qdisc *child = xchg(&q->queues[i], &noop_qdisc);
if (child != &noop_qdisc)
qdisc_destroy(child);
}
sch_tree_unlock(sch);
for (i=0; i<=TC_PRIO_MAX; i++) {
int band = q->prio2band[i];
if (q->queues[band] == &noop_qdisc) {
struct Qdisc *child;
child = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops);
if (child) {
sch_tree_lock(sch);
child = xchg(&q->queues[band], child);
if (child != &noop_qdisc)
qdisc_destroy(child);
sch_tree_unlock(sch);
}
}
}
return 0;
}
static void attach_one_default_qdisc(struct net_device *dev,
struct netdev_queue *dev_queue,
void *_unused)
{
struct Qdisc *qdisc;
const struct Qdisc_ops *ops = default_qdisc_ops;
if (dev->priv_flags & IFF_NO_QUEUE)
ops = &noqueue_qdisc_ops;
qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT);
if (!qdisc) {
netdev_info(dev, "activation failed\n");
return;
}
if (!netif_is_multiqueue(dev))
qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
dev_queue->qdisc_sleeping = qdisc;
}
static void attach_default_qdiscs(struct net_device *dev)
{
struct netdev_queue *txq;
struct Qdisc *qdisc;
txq = netdev_get_tx_queue(dev, 0);
if (!netif_is_multiqueue(dev) ||
dev->priv_flags & IFF_NO_QUEUE) {
netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
dev->qdisc = txq->qdisc_sleeping;
atomic_inc(&dev->qdisc->refcnt);
} else {
qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT);
if (qdisc) {
dev->qdisc = qdisc;
qdisc->ops->attach(qdisc);
}
}
}
void dev_activate(struct net_device *dev)
{
/* No queueing discipline is attached to device;
create default one i.e. pfifo_fast for devices,
which need queueing and noqueue_qdisc for
virtual interfaces
*/
if (dev->qdisc_sleeping == &noop_qdisc) {
struct Qdisc *qdisc;
if (dev->tx_queue_len) {
qdisc = qdisc_create_dflt(dev, &pfifo_fast_ops,
TC_H_ROOT);
if (qdisc == NULL) {
printk(KERN_INFO "%s: activation failed\n", dev->name);
return;
}
write_lock(&qdisc_tree_lock);
list_add_tail(&qdisc->list, &dev->qdisc_list);
write_unlock(&qdisc_tree_lock);
} else {
qdisc = &noqueue_qdisc;
}
write_lock(&qdisc_tree_lock);
dev->qdisc_sleeping = qdisc;
write_unlock(&qdisc_tree_lock);
}
if (!netif_carrier_ok(dev))
/* Delay activation until next carrier-on event */
return;
spin_lock_bh(&dev->queue_lock);
rcu_assign_pointer(dev->qdisc, dev->qdisc_sleeping);
if (dev->qdisc != &noqueue_qdisc) {
dev->trans_start = jiffies;
dev_watchdog_up(dev);
}
spin_unlock_bh(&dev->queue_lock);
}
static int mq_init(struct Qdisc *sch, struct nlattr *opt)
{
struct net_device *dev = qdisc_dev(sch);
struct mq_sched *priv = qdisc_priv(sch);
struct netdev_queue *dev_queue;
struct Qdisc *qdisc;
unsigned int ntx;
if (sch->parent != TC_H_ROOT)
return -EOPNOTSUPP;
if (!netif_is_multiqueue(dev))
return -EOPNOTSUPP;
/* pre-allocate qdiscs, attachment can't fail */
priv->qdiscs = kcalloc(dev->num_tx_queues, sizeof(priv->qdiscs[0]),
GFP_KERNEL);
if (priv->qdiscs == NULL)
return -ENOMEM;
for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
dev_queue = netdev_get_tx_queue(dev, ntx);
qdisc = qdisc_create_dflt(dev, dev_queue, &pfifo_fast_ops,
TC_H_MAKE(TC_H_MAJ(sch->handle),
TC_H_MIN(ntx + 1)));
if (qdisc == NULL)
goto err;
qdisc->flags |= TCQ_F_CAN_BYPASS;
priv->qdiscs[ntx] = qdisc;
}
sch->flags |= TCQ_F_MQROOT;
return 0;
err:
mq_destroy(sch);
return -ENOMEM;
}
static int mqprio_init(struct Qdisc *sch, struct nlattr *opt)
{
struct net_device *dev = qdisc_dev(sch);
struct mqprio_sched *priv = qdisc_priv(sch);
struct netdev_queue *dev_queue;
struct Qdisc *qdisc;
int i, err = -EOPNOTSUPP;
struct tc_mqprio_qopt *qopt = NULL;
BUILD_BUG_ON(TC_MAX_QUEUE != TC_QOPT_MAX_QUEUE);
BUILD_BUG_ON(TC_BITMASK != TC_QOPT_BITMASK);
if (sch->parent != TC_H_ROOT)
return -EOPNOTSUPP;
if (!netif_is_multiqueue(dev))
return -EOPNOTSUPP;
if (!opt || nla_len(opt) < sizeof(*qopt))
return -EINVAL;
qopt = nla_data(opt);
if (mqprio_parse_opt(dev, qopt))
return -EINVAL;
/* pre-allocate qdisc, attachment can't fail */
priv->qdiscs = kcalloc(dev->num_tx_queues, sizeof(priv->qdiscs[0]),
GFP_KERNEL);
if (priv->qdiscs == NULL) {
err = -ENOMEM;
goto err;
}
for (i = 0; i < dev->num_tx_queues; i++) {
dev_queue = netdev_get_tx_queue(dev, i);
qdisc = qdisc_create_dflt(dev, dev_queue, &pfifo_fast_ops,
TC_H_MAKE(TC_H_MAJ(sch->handle),
TC_H_MIN(i + 1)));
if (qdisc == NULL) {
err = -ENOMEM;
goto err;
}
qdisc->flags |= TCQ_F_CAN_BYPASS;
priv->qdiscs[i] = qdisc;
}
netdev_set_num_tc(dev, qopt->num_tc);
for (i = 0; i < qopt->num_tc; i++)
netdev_set_tc_queue(dev, i,
qopt->count[i], qopt->offset[i]);
/* Always use supplied priority mappings */
for (i = 0; i < TC_BITMASK + 1; i++)
netdev_set_prio_tc_map(dev, i, qopt->prio_tc_map[i]);
sch->flags |= TCQ_F_MQROOT;
return 0;
err:
mqprio_destroy(sch);
return err;
}
static int prio_tune(struct Qdisc *sch, struct nlattr *opt)
{
struct prio_sched_data *q = qdisc_priv(sch);
struct tc_prio_qopt *qopt;
int i;
int flow_change = 0;
if (nla_len(opt) < sizeof(*qopt))
return -EINVAL;
qopt = nla_data(opt);
if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < 2)
return -EINVAL;
for (i = 0; i <= TC_PRIO_MAX; i++) {
if (qopt->priomap[i] >= qopt->bands)
return -EINVAL;
}
sch_tree_lock(sch);
if (q->enable_flow != qopt->enable_flow) {
q->enable_flow = qopt->enable_flow;
flow_change = 1;
}
q->bands = qopt->bands;
memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);
for (i = q->bands; i < TCQ_PRIO_BANDS; i++) {
struct Qdisc *child = q->queues[i];
q->queues[i] = &noop_qdisc;
if (child != &noop_qdisc) {
qdisc_tree_decrease_qlen(child, child->q.qlen);
qdisc_destroy(child);
}
}
sch_tree_unlock(sch);
for (i = 0; i < q->bands; i++) {
if (q->queues[i] == &noop_qdisc) {
struct Qdisc *child, *old;
child = qdisc_create_dflt(sch->dev_queue,
&pfifo_qdisc_ops,
TC_H_MAKE(sch->handle, i + 1));
if (child) {
sch_tree_lock(sch);
old = q->queues[i];
q->queues[i] = child;
if (old != &noop_qdisc) {
qdisc_tree_decrease_qlen(old,
old->q.qlen);
qdisc_destroy(old);
}
sch_tree_unlock(sch);
}
}
}
/* Schedule qdisc when flow re-enabled */
if (flow_change && q->enable_flow) {
if (!test_bit(__QDISC_STATE_DEACTIVATED,
&sch->state))
__netif_schedule(qdisc_root(sch));
}
return 0;
}
static int fifo_dump(struct Qdisc *sch, struct sk_buff *skb)
{
struct tc_fifo_qopt opt = { .limit = sch->limit };
if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
goto nla_put_failure;
return skb->len;
nla_put_failure:
return -1;
}
struct Qdisc_ops pfifo_qdisc_ops __read_mostly = {
.id = "pfifo",
.priv_size = 0,
.enqueue = pfifo_enqueue,
.dequeue = qdisc_dequeue_head,
.peek = qdisc_peek_head,
.init = fifo_init,
.reset = qdisc_reset_queue,
.change = fifo_init,
.dump = fifo_dump,
.owner = THIS_MODULE,
};
EXPORT_SYMBOL(pfifo_qdisc_ops);
struct Qdisc_ops bfifo_qdisc_ops __read_mostly = {
.id = "bfifo",
.priv_size = 0,
.enqueue = bfifo_enqueue,
.dequeue = qdisc_dequeue_head,
.peek = qdisc_peek_head,
.init = fifo_init,
.reset = qdisc_reset_queue,
.change = fifo_init,
.dump = fifo_dump,
.owner = THIS_MODULE,
};
EXPORT_SYMBOL(bfifo_qdisc_ops);
struct Qdisc_ops pfifo_head_drop_qdisc_ops __read_mostly = {
.id = "pfifo_head_drop",
.priv_size = 0,
.enqueue = pfifo_tail_enqueue,
.dequeue = qdisc_dequeue_head,
.peek = qdisc_peek_head,
.init = fifo_init,
.reset = qdisc_reset_queue,
.change = fifo_init,
.dump = fifo_dump,
.owner = THIS_MODULE,
};
/* Pass size change message down to embedded FIFO */
int fifo_set_limit(struct Qdisc *q, unsigned int limit)
{
struct nlattr *nla;
int ret = -ENOMEM;
/* Hack to avoid sending change message to non-FIFO */
if (strncmp(q->ops->id + 1, "fifo", 4) != 0)
return 0;
nla = kmalloc(nla_attr_size(sizeof(struct tc_fifo_qopt)), GFP_KERNEL);
if (nla) {
nla->nla_type = RTM_NEWQDISC;
nla->nla_len = nla_attr_size(sizeof(struct tc_fifo_qopt));
((struct tc_fifo_qopt *)nla_data(nla))->limit = limit;
ret = q->ops->change(q, nla);
kfree(nla);
}
return ret;
}
EXPORT_SYMBOL(fifo_set_limit);
struct Qdisc *fifo_create_dflt(struct Qdisc *sch, struct Qdisc_ops *ops,
unsigned int limit)
{
struct Qdisc *q;
int err = -ENOMEM;
q = qdisc_create_dflt(sch->dev_queue, ops, TC_H_MAKE(sch->handle, 1));
if (q) {
err = fifo_set_limit(q, limit);
if (err < 0) {
qdisc_destroy(q);
q = NULL;
}
}
return q ? : ERR_PTR(err);
}
EXPORT_SYMBOL(fifo_create_dflt);
static int prio_tune(struct Qdisc *sch, struct nlattr *opt)
{
struct prio_sched_data *q = qdisc_priv(sch);
struct tc_prio_qopt *qopt;
struct nlattr *tb[TCA_PRIO_MAX + 1] = {0};
int err;
int i;
qopt = nla_data(opt);
if (nla_len(opt) < sizeof(*qopt))
return -1;
if (nla_len(opt) >= sizeof(*qopt) + sizeof(struct nlattr)) {
err = nla_parse_nested(tb, TCA_PRIO_MAX,
(struct nlattr *) (qopt + 1), NULL);
if (err < 0)
return err;
}
q->bands = qopt->bands;
/* If we're multiqueue, make sure the number of incoming bands
* matches the number of queues on the device we're associating with.
* If the number of bands requested is zero, then set q->bands to
* dev->egress_subqueue_count. Also, the root qdisc must be the
* only one that is enabled for multiqueue, since it's the only one
* that interacts with the underlying device.
*/
q->mq = nla_get_flag(tb[TCA_PRIO_MQ]);
if (q->mq) {
if (sch->parent != TC_H_ROOT)
return -EINVAL;
if (netif_is_multiqueue(sch->dev)) {
if (q->bands == 0)
q->bands = sch->dev->egress_subqueue_count;
else if (q->bands != sch->dev->egress_subqueue_count)
return -EINVAL;
} else
return -EOPNOTSUPP;
}
if (q->bands > TCQ_PRIO_BANDS || q->bands < 2)
return -EINVAL;
for (i=0; i<=TC_PRIO_MAX; i++) {
if (qopt->priomap[i] >= q->bands)
return -EINVAL;
}
sch_tree_lock(sch);
memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);
for (i=q->bands; i<TCQ_PRIO_BANDS; i++) {
struct Qdisc *child = xchg(&q->queues[i], &noop_qdisc);
if (child != &noop_qdisc) {
qdisc_tree_decrease_qlen(child, child->q.qlen);
qdisc_destroy(child);
}
}
sch_tree_unlock(sch);
for (i=0; i<q->bands; i++) {
if (q->queues[i] == &noop_qdisc) {
struct Qdisc *child;
child = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops,
TC_H_MAKE(sch->handle, i + 1));
if (child) {
sch_tree_lock(sch);
child = xchg(&q->queues[i], child);
if (child != &noop_qdisc) {
qdisc_tree_decrease_qlen(child,
child->q.qlen);
qdisc_destroy(child);
}
sch_tree_unlock(sch);
}
}
}
return 0;
}
static int
nm_os_catch_qdisc(struct netmap_generic_adapter *gna, int intercept)
{
struct netmap_adapter *na = &gna->up.up;
struct ifnet *ifp = netmap_generic_getifp(gna);
struct nm_generic_qdisc *qdiscopt = NULL;
struct Qdisc *fqdisc = NULL;
struct nlattr *nla = NULL;
struct netdev_queue *txq;
unsigned int i;
if (!gna->txqdisc) {
return 0;
}
if (intercept) {
nla = kmalloc(nla_attr_size(sizeof(*qdiscopt)),
GFP_KERNEL);
if (!nla) {
D("Failed to allocate netlink attribute");
return ENOMEM;
}
nla->nla_type = RTM_NEWQDISC;
nla->nla_len = nla_attr_size(sizeof(*qdiscopt));
qdiscopt = (struct nm_generic_qdisc *)nla_data(nla);
memset(qdiscopt, 0, sizeof(*qdiscopt));
qdiscopt->limit = na->num_tx_desc;
}
if (ifp->flags & IFF_UP) {
dev_deactivate(ifp);
}
/* Replace the current qdiscs with our own. */
for (i = 0; i < ifp->real_num_tx_queues; i++) {
struct Qdisc *nqdisc = NULL;
struct Qdisc *oqdisc;
int err;
txq = netdev_get_tx_queue(ifp, i);
if (intercept) {
/* This takes a refcount to netmap module, alloc the
* qdisc and calls the init() op with NULL netlink
* attribute. */
nqdisc = qdisc_create_dflt(
#ifndef NETMAP_LINUX_QDISC_CREATE_DFLT_3ARGS
ifp,
#endif /* NETMAP_LINUX_QDISC_CREATE_DFLT_3ARGS */
txq, &generic_qdisc_ops,
TC_H_UNSPEC);
if (!nqdisc) {
D("Failed to create qdisc");
goto qdisc_create;
}
fqdisc = fqdisc ?: nqdisc;
/* Call the change() op passing a valid netlink
* attribute. This is used to set the queue idx. */
qdiscopt->qidx = i;
err = nqdisc->ops->change(nqdisc, nla);
if (err) {
D("Failed to init qdisc");
goto qdisc_create;
}
}
oqdisc = dev_graft_qdisc(txq, nqdisc);
/* We can call this also with
* odisc == &noop_qdisc, since the noop
* qdisc has the TCQ_F_BUILTIN flag set,
* and so qdisc_destroy will skip it. */
qdisc_destroy(oqdisc);
}
kfree(nla);
if (ifp->qdisc) {
qdisc_destroy(ifp->qdisc);
}
if (intercept) {
atomic_inc(&fqdisc->refcnt);
ifp->qdisc = fqdisc;
} else {
ifp->qdisc = &noop_qdisc;
}
if (ifp->flags & IFF_UP) {
dev_activate(ifp);
}
return 0;
qdisc_create:
if (nla) {
kfree(nla);
}
nm_os_catch_qdisc(gna, 0);
return -1;
}
static int mqprio_init(struct Qdisc *sch, struct nlattr *opt)
{
struct net_device *dev = qdisc_dev(sch);
struct mqprio_sched *priv = qdisc_priv(sch);
struct netdev_queue *dev_queue;
struct Qdisc *qdisc;
int i, err = -EOPNOTSUPP;
struct tc_mqprio_qopt *qopt = NULL;
BUILD_BUG_ON(TC_MAX_QUEUE != TC_QOPT_MAX_QUEUE);
BUILD_BUG_ON(TC_BITMASK != TC_QOPT_BITMASK);
if (sch->parent != TC_H_ROOT)
return -EOPNOTSUPP;
if (!netif_is_multiqueue(dev))
return -EOPNOTSUPP;
if (nla_len(opt) < sizeof(*qopt))
return -EINVAL;
qopt = nla_data(opt);
if (mqprio_parse_opt(dev, qopt))
return -EINVAL;
/* pre-allocate qdisc, attachment can't fail */
priv->qdiscs = kcalloc(dev->num_tx_queues, sizeof(priv->qdiscs[0]),
GFP_KERNEL);
if (priv->qdiscs == NULL) {
err = -ENOMEM;
goto err;
}
for (i = 0; i < dev->num_tx_queues; i++) {
dev_queue = netdev_get_tx_queue(dev, i);
qdisc = qdisc_create_dflt(dev_queue, &pfifo_fast_ops,
TC_H_MAKE(TC_H_MAJ(sch->handle),
TC_H_MIN(i + 1)));
if (qdisc == NULL) {
err = -ENOMEM;
goto err;
}
priv->qdiscs[i] = qdisc;
}
/* If the mqprio options indicate that hardware should own
* the queue mapping then run ndo_setup_tc otherwise use the
* supplied and verified mapping
*/
if (qopt->hw) {
priv->hw_owned = 1;
err = dev->netdev_ops->ndo_setup_tc(dev, qopt->num_tc);
if (err)
goto err;
} else {
netdev_set_num_tc(dev, qopt->num_tc);
for (i = 0; i < qopt->num_tc; i++)
netdev_set_tc_queue(dev, i,
qopt->count[i], qopt->offset[i]);
}
/* Always use supplied priority mappings */
for (i = 0; i < TC_BITMASK + 1; i++)
netdev_set_prio_tc_map(dev, i, qopt->prio_tc_map[i]);
sch->flags |= TCQ_F_MQROOT;
return 0;
err:
mqprio_destroy(sch);
return err;
}
