static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
{
struct fq_sched_data *q = qdisc_priv(sch);
struct fq_flow *f;
if (unlikely(sch->q.qlen >= sch->limit))
return qdisc_drop(skb, sch);
f = fq_classify(skb, q);
if (unlikely(f->qlen >= q->flow_plimit && f != &q->internal)) {
q->stat_flows_plimit++;
return qdisc_drop(skb, sch);
}
f->qlen++;
flow_queue_add(f, skb);
if (skb_is_retransmit(skb))
q->stat_tcp_retrans++;
sch->qstats.backlog += qdisc_pkt_len(skb);
if (fq_flow_is_detached(f)) {
fq_flow_add_tail(&q->new_flows, f);
if (q->quantum > f->credit)
f->credit = q->quantum;
q->inactive_flows--;
qdisc_unthrottled(sch);
}
if (unlikely(f == &q->internal)) {
q->stat_internal_packets++;
qdisc_unthrottled(sch);
}
sch->q.qlen++;
return NET_XMIT_SUCCESS;
}
static struct sk_buff *netem_dequeue(struct Qdisc *sch)
{
struct netem_sched_data *q = qdisc_priv(sch);
struct sk_buff *skb;
if (qdisc_is_throttled(sch))
return NULL;
tfifo_dequeue:
skb = qdisc_peek_head(sch);
if (skb) {
const struct netem_skb_cb *cb = netem_skb_cb(skb);
/* if more time remaining? */
if (cb->time_to_send <= psched_get_time()) {
__skb_unlink(skb, &sch->q);
sch->qstats.backlog -= qdisc_pkt_len(skb);
#ifdef CONFIG_NET_CLS_ACT
/*
* If it's at ingress let's pretend the delay is
* from the network (tstamp will be updated).
*/
if (G_TC_FROM(skb->tc_verd) & AT_INGRESS)
skb->tstamp.tv64 = 0;
#endif
if (q->qdisc) {
int err = qdisc_enqueue(skb, q->qdisc);
if (unlikely(err != NET_XMIT_SUCCESS)) {
if (net_xmit_drop_count(err)) {
sch->qstats.drops++;
qdisc_tree_decrease_qlen(sch, 1);
}
}
goto tfifo_dequeue;
}
deliver:
qdisc_unthrottled(sch);
qdisc_bstats_update(sch, skb);
return skb;
}
if (q->qdisc) {
skb = q->qdisc->ops->dequeue(q->qdisc);
if (skb)
goto deliver;
}
qdisc_watchdog_schedule(&q->watchdog, cb->time_to_send);
}
if (q->qdisc) {
skb = q->qdisc->ops->dequeue(q->qdisc);
if (skb)
goto deliver;
}
return NULL;
}
static struct sk_buff *tbf_dequeue(struct Qdisc *sch)
{
struct tbf_sched_data *q = qdisc_priv(sch);
struct sk_buff *skb;
skb = q->qdisc->ops->peek(q->qdisc);
if (skb) {
psched_time_t now;
long toks;
long ptoks = 0;
unsigned int len = qdisc_pkt_len(skb);
now = psched_get_time();
toks = psched_tdiff_bounded(now, q->t_c, q->buffer);
if (q->P_tab) {
ptoks = toks + q->ptokens;
if (ptoks > (long)q->mtu)
ptoks = q->mtu;
ptoks -= L2T_P(q, len);
}
toks += q->tokens;
if (toks > (long)q->buffer)
toks = q->buffer;
toks -= L2T(q, len);
if ((toks|ptoks) >= 0) {
skb = qdisc_dequeue_peeked(q->qdisc);
if (unlikely(!skb))
return NULL;
q->t_c = now;
q->tokens = toks;
q->ptokens = ptoks;
sch->q.qlen--;
qdisc_unthrottled(sch);
qdisc_bstats_update(sch, skb);
return skb;
}
qdisc_watchdog_schedule(&q->watchdog,
now + max_t(long, -toks, -ptoks));
/* Maybe we have a shorter packet in the queue,
which can be sent now. It sounds cool,
but, however, this is wrong in principle.
We MUST NOT reorder packets under these circumstances.
Really, if we split the flow into independent
subflows, it would be a very good solution.
This is the main idea of all FQ algorithms
(cf. CSZ, HPFQ, HFSC)
*/
sch->qstats.overlimits++;
}
return NULL;
}
static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
{
struct fq_sched_data *q = qdisc_priv(sch);
struct fq_flow *f;
if (unlikely(sch->q.qlen >= sch->limit))
return qdisc_drop(skb, sch);
f = fq_classify(skb, q);
if (unlikely(f->qlen >= q->flow_plimit && f != &q->internal)) {
q->stat_flows_plimit++;
return qdisc_drop(skb, sch);
}
f->qlen++;
if (skb_is_retransmit(skb))
q->stat_tcp_retrans++;
sch->qstats.backlog += qdisc_pkt_len(skb);
if (fq_flow_is_detached(f)) {
fq_flow_add_tail(&q->new_flows, f);
if (time_after(jiffies, f->age + q->flow_refill_delay))
f->credit = max_t(u32, f->credit, q->quantum);
q->inactive_flows--;
qdisc_unthrottled(sch);
}
/* Note: this overwrites f->age */
flow_queue_add(f, skb);
if (unlikely(f == &q->internal)) {
q->stat_internal_packets++;
qdisc_unthrottled(sch);
}
sch->q.qlen++;
return NET_XMIT_SUCCESS;
}
static struct sk_buff *netem_dequeue(struct Qdisc *sch)
{
struct netem_sched_data *q = qdisc_priv(sch);
struct sk_buff *skb;
if (qdisc_is_throttled(sch))
return NULL;
skb = q->qdisc->ops->peek(q->qdisc);
if (skb) {
const struct netem_skb_cb *cb = netem_skb_cb(skb);
psched_time_t now = psched_get_time();
/* if more time remaining? */
if (cb->time_to_send <= now) {
skb = qdisc_dequeue_peeked(q->qdisc);
if (unlikely(!skb))
return NULL;
#ifdef CONFIG_NET_CLS_ACT
/*
* If it's at ingress let's pretend the delay is
* from the network (tstamp will be updated).
*/
if (G_TC_FROM(skb->tc_verd) & AT_INGRESS)
skb->tstamp.tv64 = 0;
#endif
sch->q.qlen--;
qdisc_unthrottled(sch);
qdisc_bstats_update(sch, skb);
return skb;
}
qdisc_watchdog_schedule(&q->watchdog, cb->time_to_send);
}
return NULL;
}
static struct sk_buff *fq_dequeue(struct Qdisc *sch)
{
struct fq_sched_data *q = qdisc_priv(sch);
u64 now = ktime_to_ns(ktime_get());
struct fq_flow_head *head;
struct sk_buff *skb;
struct fq_flow *f;
u32 rate;
skb = fq_dequeue_head(sch, &q->internal);
if (skb)
goto out;
fq_check_throttled(q, now);
begin:
head = &q->new_flows;
if (!head->first) {
head = &q->old_flows;
if (!head->first) {
if (q->time_next_delayed_flow != ~0ULL)
qdisc_watchdog_schedule_ns(&q->watchdog,
q->time_next_delayed_flow);
return NULL;
}
}
f = head->first;
if (f->credit <= 0) {
f->credit += q->quantum;
head->first = f->next;
fq_flow_add_tail(&q->old_flows, f);
goto begin;
}
if (unlikely(f->head && now < f->time_next_packet)) {
head->first = f->next;
fq_flow_set_throttled(q, f);
goto begin;
}
skb = fq_dequeue_head(sch, f);
if (!skb) {
head->first = f->next;
/* force a pass through old_flows to prevent starvation */
if ((head == &q->new_flows) && q->old_flows.first) {
fq_flow_add_tail(&q->old_flows, f);
} else {
fq_flow_set_detached(f);
f->age = jiffies;
q->inactive_flows++;
}
goto begin;
}
prefetch(&skb->end);
f->time_next_packet = now;
f->credit -= qdisc_pkt_len(skb);
if (f->credit > 0 || !q->rate_enable)
goto out;
rate = q->flow_max_rate;
if (skb->sk && skb->sk->sk_state != TCP_TIME_WAIT)
rate = min(skb->sk->sk_pacing_rate, rate);
if (rate != ~0U) {
u32 plen = max(qdisc_pkt_len(skb), q->quantum);
u64 len = (u64)plen * NSEC_PER_SEC;
if (likely(rate))
do_div(len, rate);
/* Since socket rate can change later,
* clamp the delay to 125 ms.
* TODO: maybe segment the too big skb, as in commit
* e43ac79a4bc ("sch_tbf: segment too big GSO packets")
*/
if (unlikely(len > 125 * NSEC_PER_MSEC)) {
len = 125 * NSEC_PER_MSEC;
q->stat_pkts_too_long++;
}
f->time_next_packet = now + len;
}
out:
qdisc_bstats_update(sch, skb);
qdisc_unthrottled(sch);
return skb;
}
static struct sk_buff *wfq_dequeue(struct Qdisc *sch)
{
struct wfq_sched_data *q = qdisc_priv(sch);
int i, weight;
struct wfq_class *cl = NULL;
u64 min_time;
struct sk_buff *skb = NULL;
struct sk_buff *next_pkt = NULL;
unsigned int len;
s64 bucket_ns = (s64)l2t_ns(&q->rate, wfq_bucket_bytes);
s64 result, now;
int prio = prio_schedule(q);
if (prio < 0)
return NULL;
/* Find the active queue with the smallest head finish time */
for (i = 0; i < wfq_max_queues; i++)
{
if (q->queues[i].prio != prio || q->queues[i].len_bytes == 0 )
continue;
if (!cl || wfq_time_before(q->queues[i].head_fin_time,
min_time))
{
cl = &q->queues[i];
min_time = cl->head_fin_time;
}
}
/* get head packet */
skb = cl->qdisc->ops->peek(cl->qdisc);
if (unlikely(!skb))
return NULL;
len = skb_size(skb);
now = ktime_get_ns();
result = tbf_schedule(len, q, now);
/* We don't have enough tokens */
if (result < 0)
{
/* For hrtimer absolute mode, we use now + t */
qdisc_watchdog_schedule_ns(&q->watchdog, now - result, true);
qdisc_qstats_overlimit(sch);
return NULL;
}
skb = qdisc_dequeue_peeked(cl->qdisc);
if (unlikely(!skb))
return NULL;
q->sum_len_bytes -= len;
sch->q.qlen--;
cl->len_bytes -= len;
q->prio_len_bytes[prio] -= len;
/* Set the head_fin_time for the remaining head packet */
if (cl->len_bytes > 0)
{
/* Get the current head packet */
next_pkt = cl->qdisc->ops->peek(cl->qdisc);
weight = wfq_queue_weight[cl->id];
if (likely(next_pkt && weight))
{
len = skb_size(next_pkt);
cl->head_fin_time += div_u64((u64)len, (u32)weight);
if (wfq_time_before(q->virtual_time[prio],
cl->head_fin_time))
q->virtual_time[prio] = cl->head_fin_time;
}
}
/* Bucket */
q->time_ns = now;
q->tokens = min_t(s64, result, bucket_ns);
qdisc_unthrottled(sch);
qdisc_bstats_update(sch, skb);
/* TCN */
if (wfq_ecn_scheme == wfq_tcn)
tcn_marking(skb);
/* CoDel */
else if (wfq_ecn_scheme == wfq_codel)
codel_marking(skb, cl);
/* dequeue equeue length based ECN marking */
else if (wfq_enable_dequeue_ecn == wfq_enable)
wfq_qlen_marking(skb, q, cl);
return skb;
}