/* Azure hosts don't support non-TCP port numbers in hashing yet. We compute
* hash for non-TCP traffic with only IP numbers.
*/
static inline u32 netvsc_get_hash(struct sk_buff *skb, struct sock *sk)
{
struct flow_keys flow;
u32 hash;
static u32 hashrnd __read_mostly;
net_get_random_once(&hashrnd, sizeof(hashrnd));
if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
return 0;
if (flow.basic.ip_proto == IPPROTO_TCP) {
return skb_get_hash(skb);
} else {
if (flow.basic.n_proto == htons(ETH_P_IP))
hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
else if (flow.basic.n_proto == htons(ETH_P_IPV6))
hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
else
hash = 0;
skb_set_hash(skb, hash, PKT_HASH_TYPE_L3);
}
return hash;
}
static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed)
{
const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data;
return jhash2((u32 *)&cmp_arg->cookie,
sizeof(cmp_arg->cookie) / sizeof(u32), seed);
}
static u32 ip4_obj_hashfn(const void *data, u32 len, u32 seed)
{
const struct inet_frag_queue *fq = data;
return jhash2((const u32 *)&fq->key.v4,
sizeof(struct frag_v4_compare_key) / sizeof(u32), seed);
}
static u32 flow_hash_code(struct flowi *key, int cpu)
{
u32 *k = (u32 *) key;
return (jhash2(k, (sizeof(*key) / sizeof(u32)), flow_hash_rnd(cpu)) &
(flow_hash_size - 1));
}
static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed)
{
const struct nfp_fl_payload *flower_entry = data;
return jhash2((u32 *)&flower_entry->tc_flower_cookie,
sizeof(flower_entry->tc_flower_cookie) / sizeof(u32),
seed);
}
static unsigned int recent_entry_hash6(const union nf_inet_addr *addr)
{
if (!hash_rnd_initted) {
get_random_bytes(&hash_rnd, sizeof(hash_rnd));
hash_rnd_initted = true;
}
return jhash2((u32 *)addr->ip6, ARRAY_SIZE(addr->ip6), hash_rnd) &
(ip_list_hash_size - 1);
}
unsigned int sockunion_hash(const union sockunion *su)
{
switch (sockunion_family(su)) {
case AF_INET:
return jhash_1word(su->sin.sin_addr.s_addr, 0);
case AF_INET6:
return jhash2(su->sin6.sin6_addr.s6_addr32,
ZEBRA_NUM_OF(su->sin6.sin6_addr.s6_addr32), 0);
}
return 0;
}
static inline unsigned int
connlimit_iphash6(const union nf_inet_addr *addr,
const union nf_inet_addr *mask)
{
union nf_inet_addr res;
unsigned int i;
for (i = 0; i < ARRAY_SIZE(addr->ip6); ++i)
res.ip6[i] = addr->ip6[i] & mask->ip6[i];
return jhash2((u32 *)res.ip6, ARRAY_SIZE(res.ip6), connlimit_rnd) & 0xFF;
}
static u32 nf_nat_bysource_hash(const void *data, u32 len, u32 seed)
{
const struct nf_conntrack_tuple *t;
const struct nf_conn *ct = data;
t = &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple;
/* Original src, to ensure we map it consistently if poss. */
seed ^= net_hash_mix(nf_ct_net(ct));
return jhash2((const u32 *)&t->src, sizeof(t->src) / sizeof(u32),
t->dst.protonum ^ seed);
}
static u32 hash_v6(const struct sk_buff *skb)
{
const struct ipv6hdr *ip6h = ipv6_hdr(skb);
__be32 addr[4];
addr[0] = ip6h->saddr.s6_addr32[0] ^ ip6h->daddr.s6_addr32[0];
addr[1] = ip6h->saddr.s6_addr32[1] ^ ip6h->daddr.s6_addr32[1];
addr[2] = ip6h->saddr.s6_addr32[2] ^ ip6h->daddr.s6_addr32[2];
addr[3] = ip6h->saddr.s6_addr32[3] ^ ip6h->daddr.s6_addr32[3];
return jhash2((__force u32 *)addr, ARRAY_SIZE(addr), jhash_initval);
}
static u32 __ipv6_select_ident(const struct in6_addr *addr)
{
u32 newid, oldid, hash = jhash2((u32 *)addr, 4, hashidentrnd);
u32 *pid = &ipv6_fragmentation_id[hash % FID_HASH_SZ];
do {
oldid = *pid;
newid = oldid + 1;
if (!(hash + newid))
newid++;
} while (cmpxchg(pid, oldid, newid) != oldid);
return hash + newid;
}
static inline unsigned int
connlimit_iphash6(const union nf_inet_addr *addr,
const union nf_inet_addr *mask)
{
union nf_inet_addr res;
unsigned int i;
if (unlikely(!connlimit_rnd_inited)) {
get_random_bytes(&connlimit_rnd, sizeof(connlimit_rnd));
connlimit_rnd_inited = true;
}
for (i = 0; i < ARRAY_SIZE(addr->ip6); ++i)
res.ip6[i] = addr->ip6[i] & mask->ip6[i];
return jhash2((u32 *)res.ip6, ARRAY_SIZE(res.ip6), connlimit_rnd) & 0xFF;
}
static uint32_t hash_ipv6(struct in6_addr *addr)
{
return jhash2(addr->s6_addr32, sizeof(*addr) / 4, 0);
}
static inline u_int32_t
xt_cluster_hash_ipv6(const void *ip, const struct xt_cluster_match_info *info)
{
return jhash2(ip, NF_CT_TUPLE_L3SIZE / sizeof(__u32), info->hash_seed);
}
static __always_inline u32 __flow_hash_words(const u32 *words, u32 length,
u32 keyval)
{
return jhash2(words, length, keyval);
}
static unsigned sfq_hash(struct sfq_sched_data *q, struct sk_buff *skb)
{
struct sfq_packet_info info;
u32 pert = q->perturbation;
unsigned mask = (1<<q->hash_divisor) - 1;
#ifdef CONFIG_NET_SCH_SFQ_NFCT
enum ip_conntrack_info ctinfo;
struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
#endif
switch (skb->protocol) {
case __constant_htons(ETH_P_IP):
{
const struct iphdr *iph = ip_hdr(skb);
info.dst = iph->daddr;
info.src = iph->saddr;
if (!(iph->frag_off&htons(IP_MF|IP_OFFSET)) &&
(iph->protocol == IPPROTO_TCP ||
iph->protocol == IPPROTO_UDP ||
iph->protocol == IPPROTO_UDPLITE ||
iph->protocol == IPPROTO_SCTP ||
iph->protocol == IPPROTO_DCCP ||
iph->protocol == IPPROTO_ESP))
info.proto = *(((u32*)iph) + iph->ihl);
else
info.proto = iph->protocol;
break;
}
case __constant_htons(ETH_P_IPV6):
{
struct ipv6hdr *iph = ipv6_hdr(skb);
/* Hash ipv6 addresses into a u32. This isn't ideal,
* but the code is simple. */
info.dst = jhash2(iph->daddr.s6_addr32, 4, q->perturbation);
info.src = jhash2(iph->saddr.s6_addr32, 4, q->perturbation);
if (iph->nexthdr == IPPROTO_TCP ||
iph->nexthdr == IPPROTO_UDP ||
iph->nexthdr == IPPROTO_UDPLITE ||
iph->nexthdr == IPPROTO_SCTP ||
iph->nexthdr == IPPROTO_DCCP ||
iph->nexthdr == IPPROTO_ESP)
info.proto = *(u32*)&iph[1];
else
info.proto = iph->nexthdr;
break;
}
default:
info.dst = (u32)(unsigned long)skb->dst;
info.src = (u32)(unsigned long)skb->sk;
info.proto = skb->protocol;
}
info.mark = skb->mark;
#ifdef CONFIG_NET_SCH_SFQ_NFCT
/* defaults if there is no conntrack info */
info.ctorigsrc = info.src;
info.ctorigdst = info.dst;
info.ctreplsrc = info.dst;
info.ctrepldst = info.src;
/* collect conntrack info */
if (ct && !nf_ct_is_untracked(ct)) {
if (skb->protocol == __constant_htons(ETH_P_IP)) {
info.ctorigsrc =
ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.ip;
info.ctorigdst =
ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u3.ip;
info.ctreplsrc =
ct->tuplehash[IP_CT_DIR_REPLY].tuple.src.u3.ip;
info.ctrepldst =
ct->tuplehash[IP_CT_DIR_REPLY].tuple.dst.u3.ip;
}
else if (skb->protocol == __constant_htons(ETH_P_IPV6)) {
/* Again, hash ipv6 addresses into a single u32. */
info.ctorigsrc = jhash2(
ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.ip6,
4, pert);
info.ctorigdst = jhash2(
ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u3.ip6,
4, pert);
info.ctreplsrc = jhash2(
ct->tuplehash[IP_CT_DIR_REPLY].tuple.src.u3.ip6,
4, pert);
info.ctrepldst = jhash2(
ct->tuplehash[IP_CT_DIR_REPLY].tuple.dst.u3.ip6,
4, pert);
}
}
#endif
switch (q->hash_kind) {
case TCA_SFQ_HASH_CLASSIC:
return jhash_3words(info.dst, info.src, info.proto, pert) & mask;
case TCA_SFQ_HASH_DST:
return jhash_1word(info.dst, pert) & mask;
case TCA_SFQ_HASH_SRC:
return jhash_1word(info.src, pert) & mask;
case TCA_SFQ_HASH_FWMARK:
return jhash_1word(info.mark, pert) & mask;
#ifdef CONFIG_NET_SCH_SFQ_NFCT
case TCA_SFQ_HASH_CTORIGDST:
return jhash_1word(info.ctorigdst, pert) & mask;
case TCA_SFQ_HASH_CTORIGSRC:
return jhash_1word(info.ctorigsrc, pert) & mask;
case TCA_SFQ_HASH_CTREPLDST:
return jhash_1word(info.ctrepldst, pert) & mask;
case TCA_SFQ_HASH_CTREPLSRC:
return jhash_1word(info.ctreplsrc, pert) & mask;
case TCA_SFQ_HASH_CTNATCHG:
{
if (info.ctorigdst == info.ctreplsrc)
return jhash_1word(info.ctorigsrc, pert) & mask;
return jhash_1word(info.ctreplsrc, pert) & mask;
}
#else
case TCA_SFQ_HASH_CTORIGDST:
case TCA_SFQ_HASH_CTORIGSRC:
case TCA_SFQ_HASH_CTREPLDST:
case TCA_SFQ_HASH_CTREPLSRC:
case TCA_SFQ_HASH_CTNATCHG:
if (net_ratelimit())
printk(KERN_WARNING "SFQ: Conntrack support not enabled.");
#endif
}
if (net_ratelimit())
printk(KERN_WARNING "SFQ: Unknown hash method. "
"Falling back to classic.\n");
q->hash_kind = TCA_SFQ_HASH_CLASSIC;
return jhash_3words(info.dst, info.src, info.proto, pert) & mask;
}
BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
u64, flags)
{
struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
struct perf_callchain_entry *trace;
struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
u32 max_depth = map->value_size / 8;
/* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
u32 init_nr = sysctl_perf_event_max_stack - max_depth;
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
u32 hash, id, trace_nr, trace_len;
bool user = flags & BPF_F_USER_STACK;
bool kernel = !user;
u64 *ips;
if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
return -EINVAL;
trace = get_perf_callchain(regs, init_nr, kernel, user,
sysctl_perf_event_max_stack, false, false);
if (unlikely(!trace))
/* couldn't fetch the stack trace */
return -EFAULT;
/* get_perf_callchain() guarantees that trace->nr >= init_nr
* and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth
*/
trace_nr = trace->nr - init_nr;
if (trace_nr <= skip)
/* skipping more than usable stack trace */
return -EFAULT;
trace_nr -= skip;
trace_len = trace_nr * sizeof(u64);
ips = trace->ip + skip + init_nr;
hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
id = hash & (smap->n_buckets - 1);
bucket = READ_ONCE(smap->buckets[id]);
if (bucket && bucket->hash == hash) {
if (flags & BPF_F_FAST_STACK_CMP)
return id;
if (bucket->nr == trace_nr &&
memcmp(bucket->ip, ips, trace_len) == 0)
return id;
}
/* this call stack is not in the map, try to add it */
if (bucket && !(flags & BPF_F_REUSE_STACKID))
return -EEXIST;
new_bucket = (struct stack_map_bucket *)
pcpu_freelist_pop(&smap->freelist);
if (unlikely(!new_bucket))
return -ENOMEM;
memcpy(new_bucket->ip, ips, trace_len);
new_bucket->hash = hash;
new_bucket->nr = trace_nr;
old_bucket = xchg(&smap->buckets[id], new_bucket);
if (old_bucket)
pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
return id;
}
/* For [FUTURE] fragmentation handling, we want the least-used
* src-ip/dst-ip/proto triple. Fairness doesn't come into it. Thus
* if the range specifies 1.2.3.4 ports 10000-10005 and 1.2.3.5 ports
* 1-65535, we don't do pro-rata allocation based on ports; we choose
* the ip with the lowest src-ip/dst-ip/proto usage.
*/
static void
find_best_ips_proto(const struct nf_conntrack_zone *zone,
struct nf_conntrack_tuple *tuple,
const struct nf_nat_range *range,
const struct nf_conn *ct,
enum nf_nat_manip_type maniptype)
{
union nf_inet_addr *var_ipp;
unsigned int i, max;
/* Host order */
u32 minip, maxip, j, dist;
bool full_range;
/* No IP mapping? Do nothing. */
if (!(range->flags & NF_NAT_RANGE_MAP_IPS))
return;
if (maniptype == NF_NAT_MANIP_SRC)
var_ipp = &tuple->src.u3;
else
var_ipp = &tuple->dst.u3;
/* Fast path: only one choice. */
if (nf_inet_addr_cmp(&range->min_addr, &range->max_addr)) {
*var_ipp = range->min_addr;
return;
}
if (nf_ct_l3num(ct) == NFPROTO_IPV4)
max = sizeof(var_ipp->ip) / sizeof(u32) - 1;
else
max = sizeof(var_ipp->ip6) / sizeof(u32) - 1;
/* Hashing source and destination IPs gives a fairly even
* spread in practice (if there are a small number of IPs
* involved, there usually aren't that many connections
* anyway). The consistency means that servers see the same
* client coming from the same IP (some Internet Banking sites
* like this), even across reboots.
*/
j = jhash2((u32 *)&tuple->src.u3, sizeof(tuple->src.u3) / sizeof(u32),
range->flags & NF_NAT_RANGE_PERSISTENT ?
0 : (__force u32)tuple->dst.u3.all[max] ^ zone->id);
full_range = false;
for (i = 0; i <= max; i++) {
/* If first bytes of the address are at the maximum, use the
* distance. Otherwise use the full range.
*/
if (!full_range) {
minip = ntohl((__force __be32)range->min_addr.all[i]);
maxip = ntohl((__force __be32)range->max_addr.all[i]);
dist = maxip - minip + 1;
} else {
minip = 0;
dist = ~0;
}
var_ipp->all[i] = (__force __u32)
htonl(minip + reciprocal_scale(j, dist));
if (var_ipp->all[i] != range->max_addr.all[i])
full_range = true;
if (!(range->flags & NF_NAT_RANGE_PERSISTENT))
j ^= (__force u32)tuple->dst.u3.all[i];
}
}
static inline unsigned int recent_entry_hash6(const union nf_inet_addr *addr)
{
return jhash2((u32 *)addr->ip6, ARRAY_SIZE(addr->ip6), hash_rnd) &
(ip_list_hash_size - 1);
}
static u32 ip4_key_hashfn(const void *data, u32 len, u32 seed)
{
return jhash2(data,
sizeof(struct frag_v4_compare_key) / sizeof(u32), seed);
}