/*
* If using 'internal' SFB flow classifier, hash comes from skb rxhash
* If using external classifier, hash comes from the classid.
*/
static u32 sfb_hash(const struct sk_buff *skb, u32 slot)
{
return sfb_skb_cb(skb)->hashes[slot];
}
static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch)
{
struct sfb_sched_data *q = qdisc_priv(sch);
struct Qdisc *child = q->qdisc;
int i;
u32 p_min = ~0;
u32 minqlen = ~0;
u32 r, slot, salt, sfbhash;
int ret = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
struct flow_keys keys;
if (unlikely(sch->q.qlen >= q->limit)) {
sch->qstats.overlimits++;
q->stats.queuedrop++;
goto drop;
}
if (q->rehash_interval > 0) {
unsigned long limit = q->rehash_time + q->rehash_interval;
if (unlikely(time_after(jiffies, limit))) {
sfb_swap_slot(q);
q->rehash_time = jiffies;
} else if (unlikely(!q->double_buffering && q->warmup_time > 0 &&
time_after(jiffies, limit - q->warmup_time))) {
q->double_buffering = true;
}
}
if (q->filter_list) {
/* If using external classifiers, get result and record it. */
if (!sfb_classify(skb, q, &ret, &salt))
goto other_drop;
keys.src = salt;
keys.dst = 0;
keys.ports = 0;
} else {
skb_flow_dissect(skb, &keys);
}
slot = q->slot;
sfbhash = jhash_3words((__force u32)keys.dst,
(__force u32)keys.src,
(__force u32)keys.ports,
q->bins[slot].perturbation);
if (!sfbhash)
sfbhash = 1;
sfb_skb_cb(skb)->hashes[slot] = sfbhash;
for (i = 0; i < SFB_LEVELS; i++) {
u32 hash = sfbhash & SFB_BUCKET_MASK;
struct sfb_bucket *b = &q->bins[slot].bins[i][hash];
sfbhash >>= SFB_BUCKET_SHIFT;
if (b->qlen == 0)
decrement_prob(b, q);
else if (b->qlen >= q->bin_size)
increment_prob(b, q);
if (minqlen > b->qlen)
minqlen = b->qlen;
if (p_min > b->p_mark)
p_min = b->p_mark;
}
slot ^= 1;
sfb_skb_cb(skb)->hashes[slot] = 0;
if (unlikely(minqlen >= q->max)) {
sch->qstats.overlimits++;
q->stats.bucketdrop++;
goto drop;
}
if (unlikely(p_min >= SFB_MAX_PROB)) {
/* Inelastic flow */
if (q->double_buffering) {
sfbhash = jhash_3words((__force u32)keys.dst,
(__force u32)keys.src,
(__force u32)keys.ports,
q->bins[slot].perturbation);
if (!sfbhash)
sfbhash = 1;
sfb_skb_cb(skb)->hashes[slot] = sfbhash;
for (i = 0; i < SFB_LEVELS; i++) {
u32 hash = sfbhash & SFB_BUCKET_MASK;
struct sfb_bucket *b = &q->bins[slot].bins[i][hash];
sfbhash >>= SFB_BUCKET_SHIFT;
if (b->qlen == 0)
decrement_prob(b, q);
else if (b->qlen >= q->bin_size)
increment_prob(b, q);
}
}
if (sfb_rate_limit(skb, q)) {
sch->qstats.overlimits++;
q->stats.penaltydrop++;
goto drop;
}
goto enqueue;
}
static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff **to_free)
{
struct sfb_sched_data *q = qdisc_priv(sch);
struct Qdisc *child = q->qdisc;
struct tcf_proto *fl;
int i;
u32 p_min = ~0;
u32 minqlen = ~0;
u32 r, sfbhash;
u32 slot = q->slot;
int ret = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
if (unlikely(sch->q.qlen >= q->limit)) {
qdisc_qstats_overlimit(sch);
q->stats.queuedrop++;
goto drop;
}
if (q->rehash_interval > 0) {
unsigned long limit = q->rehash_time + q->rehash_interval;
if (unlikely(time_after(jiffies, limit))) {
sfb_swap_slot(q);
q->rehash_time = jiffies;
} else if (unlikely(!q->double_buffering && q->warmup_time > 0 &&
time_after(jiffies, limit - q->warmup_time))) {
q->double_buffering = true;
}
}
fl = rcu_dereference_bh(q->filter_list);
if (fl) {
u32 salt;
/* If using external classifiers, get result and record it. */
if (!sfb_classify(skb, fl, &ret, &salt))
goto other_drop;
sfbhash = jhash_1word(salt, q->bins[slot].perturbation);
} else {
sfbhash = skb_get_hash_perturb(skb, q->bins[slot].perturbation);
}
if (!sfbhash)
sfbhash = 1;
sfb_skb_cb(skb)->hashes[slot] = sfbhash;
for (i = 0; i < SFB_LEVELS; i++) {
u32 hash = sfbhash & SFB_BUCKET_MASK;
struct sfb_bucket *b = &q->bins[slot].bins[i][hash];
sfbhash >>= SFB_BUCKET_SHIFT;
if (b->qlen == 0)
decrement_prob(b, q);
else if (b->qlen >= q->bin_size)
increment_prob(b, q);
if (minqlen > b->qlen)
minqlen = b->qlen;
if (p_min > b->p_mark)
p_min = b->p_mark;
}
slot ^= 1;
sfb_skb_cb(skb)->hashes[slot] = 0;
if (unlikely(minqlen >= q->max)) {
qdisc_qstats_overlimit(sch);
q->stats.bucketdrop++;
goto drop;
}
if (unlikely(p_min >= SFB_MAX_PROB)) {
/* Inelastic flow */
if (q->double_buffering) {
sfbhash = skb_get_hash_perturb(skb,
q->bins[slot].perturbation);
if (!sfbhash)
sfbhash = 1;
sfb_skb_cb(skb)->hashes[slot] = sfbhash;
for (i = 0; i < SFB_LEVELS; i++) {
u32 hash = sfbhash & SFB_BUCKET_MASK;
struct sfb_bucket *b = &q->bins[slot].bins[i][hash];
sfbhash >>= SFB_BUCKET_SHIFT;
if (b->qlen == 0)
decrement_prob(b, q);
else if (b->qlen >= q->bin_size)
increment_prob(b, q);
}
}
if (sfb_rate_limit(skb, q)) {
qdisc_qstats_overlimit(sch);
q->stats.penaltydrop++;
goto drop;
}
goto enqueue;
}