static struct netdev_queue *imq_select_queue(struct net_device *dev, struct sk_buff *skb) { u16 queue_index = 0; u32 hash; if (likely(dev->real_num_tx_queues == 1)) goto out; /* IMQ can be receiving ingress or engress packets. */ /* Check first for if rx_queue is set */ if (skb_rx_queue_recorded(skb)) { queue_index = skb_get_rx_queue(skb); goto out; } /* Check if socket has tx_queue set */ if (sk_tx_queue_recorded(skb->sk)) { queue_index = sk_tx_queue_get(skb->sk); goto out; } /* Try use socket hash */ if (skb->sk && skb->sk->sk_hash) { hash = skb->sk->sk_hash; queue_index = (u16)(((u64)hash * dev->real_num_tx_queues) >> 32); goto out; }
/* * Select queue for transmit. * * If a valid queue has already been assigned, then use that. * Otherwise compute tx queue based on hash and the send table. * * This is basically similar to default (__netdev_pick_tx) with the added step * of using the host send_table when no other queue has been assigned. * * TODO support XPS - but get_xps_queue not exported */ static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb, void *accel_priv, select_queue_fallback_t fallback) { unsigned int num_tx_queues = ndev->real_num_tx_queues; int q_idx = sk_tx_queue_get(skb->sk); if (q_idx < 0 || skb->ooo_okay) { /* If forwarding a packet, we use the recorded queue when * available for better cache locality. */ if (skb_rx_queue_recorded(skb)) q_idx = skb_get_rx_queue(skb); else q_idx = netvsc_get_tx_queue(ndev, skb, q_idx); } while (unlikely(q_idx >= num_tx_queues)) q_idx -= num_tx_queues; return q_idx; }
/* * Returns a Tx hash based on the given packet descriptor a Tx queues' number * to be used as a distribution range. */ u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb, unsigned int num_tx_queues) { u32 hash; u16 qoffset = 0; u16 qcount = num_tx_queues; if (skb_rx_queue_recorded(skb)) { hash = skb_get_rx_queue(skb); while (unlikely(hash >= num_tx_queues)) hash -= num_tx_queues; return hash; } if (dev->num_tc) { u8 tc = netdev_get_prio_tc_map(dev, skb->priority); qoffset = dev->tc_to_txq[tc].offset; qcount = dev->tc_to_txq[tc].count; } return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; }
u16 mpodp_select_queue(struct net_device *dev, struct sk_buff *skb #if (LINUX_VERSION_CODE > KERNEL_VERSION (3, 13, 0)) , void *accel_priv, select_queue_fallback_t fallback #endif ) { int txq; txq = (skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : smp_processor_id()); txq = txq % dev->real_num_tx_queues; return txq; #if (LINUX_VERSION_CODE > KERNEL_VERSION (3, 13, 0)) return fallback(dev, skb) % dev->real_num_tx_queues; #else return __skb_tx_hash(dev, skb, dev->real_num_tx_queues); #endif }
int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, u16 rxq_index, u32 flow_id) { struct efx_nic *efx = netdev_priv(net_dev); struct efx_channel *channel; struct efx_filter_spec spec; const __be16 *ports; __be16 ether_type; int nhoff; int rc; /* The core RPS/RFS code has already parsed and validated * VLAN, IP and transport headers. We assume they are in the * header area. */ if (skb->protocol == htons(ETH_P_8021Q)) { const struct vlan_hdr *vh = (const struct vlan_hdr *)skb->data; /* We can't filter on the IP 5-tuple and the vlan * together, so just strip the vlan header and filter * on the IP part. */ EFX_BUG_ON_PARANOID(skb_headlen(skb) < sizeof(*vh)); ether_type = vh->h_vlan_encapsulated_proto; nhoff = sizeof(struct vlan_hdr); } else { ether_type = skb->protocol; nhoff = 0; } if (ether_type != htons(ETH_P_IP) && ether_type != htons(ETH_P_IPV6)) return -EPROTONOSUPPORT; efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, rxq_index); spec.match_flags = EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT; spec.ether_type = ether_type; if (ether_type == htons(ETH_P_IP)) { const struct iphdr *ip = (const struct iphdr *)(skb->data + nhoff); EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip)); if (ip_is_fragment(ip)) return -EPROTONOSUPPORT; spec.ip_proto = ip->protocol; spec.rem_host[0] = ip->saddr; spec.loc_host[0] = ip->daddr; EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4); ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); } else { const struct ipv6hdr *ip6 = (const struct ipv6hdr *)(skb->data + nhoff); EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip6) + 4); spec.ip_proto = ip6->nexthdr; memcpy(spec.rem_host, &ip6->saddr, sizeof(ip6->saddr)); memcpy(spec.loc_host, &ip6->daddr, sizeof(ip6->daddr)); ports = (const __be16 *)(ip6 + 1); } spec.rem_port = ports[0]; spec.loc_port = ports[1]; rc = efx->type->filter_rfs_insert(efx, &spec); if (rc < 0) return rc; /* Remember this so we can check whether to expire the filter later */ efx->rps_flow_id[rc] = flow_id; channel = efx_get_channel(efx, skb_get_rx_queue(skb)); ++channel->rfs_filters_added; if (ether_type == htons(ETH_P_IP)) netif_info(efx, rx_status, efx->net_dev, "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", spec.rem_host, ntohs(ports[0]), spec.loc_host, ntohs(ports[1]), rxq_index, flow_id, rc); else netif_info(efx, rx_status, efx->net_dev, "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d]\n", (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", spec.rem_host, ntohs(ports[0]), spec.loc_host, ntohs(ports[1]), rxq_index, flow_id, rc); return rc; }