/* Azure hosts don't support non-TCP port numbers in hashing yet. We compute * hash for non-TCP traffic with only IP numbers. */ static inline u32 netvsc_get_hash(struct sk_buff *skb, struct sock *sk) { struct flow_keys flow; u32 hash; static u32 hashrnd __read_mostly; net_get_random_once(&hashrnd, sizeof(hashrnd)); if (!skb_flow_dissect_flow_keys(skb, &flow, 0)) return 0; if (flow.basic.ip_proto == IPPROTO_TCP) { return skb_get_hash(skb); } else { if (flow.basic.n_proto == htons(ETH_P_IP)) hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd); else if (flow.basic.n_proto == htons(ETH_P_IPV6)) hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd); else hash = 0; skb_set_hash(skb, hash, PKT_HASH_TYPE_L3); } return hash; }
/** * nfp_net_set_hash() - Set SKB hash data * @netdev: adapter's net_device structure * @skb: SKB to set the hash data on * @rxd: RX descriptor * * The RSS hash and hash-type are pre-pended to the packet data. * Extract and decode it and set the skb fields. */ static void nfp_net_set_hash(struct net_device *netdev, struct sk_buff *skb, struct nfp_net_rx_desc *rxd) { struct nfp_net_rx_hash *rx_hash; if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS) || !(netdev->features & NETIF_F_RXHASH)) return; rx_hash = (struct nfp_net_rx_hash *)(skb->data - sizeof(*rx_hash)); switch (be32_to_cpu(rx_hash->hash_type)) { case NFP_NET_RSS_IPV4: case NFP_NET_RSS_IPV6: case NFP_NET_RSS_IPV6_EX: skb_set_hash(skb, be32_to_cpu(rx_hash->hash), PKT_HASH_TYPE_L3); break; default: skb_set_hash(skb, be32_to_cpu(rx_hash->hash), PKT_HASH_TYPE_L4); break; } }
/* Pass a received packet up through GRO. GRO can handle pages * regardless of checksum state and skbs with a good checksum. */ static void efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, unsigned int n_frags, u8 *eh) { struct napi_struct *napi = &channel->napi_str; gro_result_t gro_result; struct efx_nic *efx = channel->efx; struct sk_buff *skb; skb = napi_get_frags(napi); if (unlikely(!skb)) { struct efx_rx_queue *rx_queue; rx_queue = efx_channel_get_rx_queue(channel); efx_free_rx_buffers(rx_queue, rx_buf, n_frags); return; } if (efx->net_dev->features & NETIF_F_RXHASH) skb_set_hash(skb, efx_rx_buf_hash(efx, eh), PKT_HASH_TYPE_L3); skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ? CHECKSUM_UNNECESSARY : CHECKSUM_NONE); for (;;) { skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, rx_buf->page, rx_buf->page_offset, rx_buf->len); rx_buf->page = NULL; skb->len += rx_buf->len; if (skb_shinfo(skb)->nr_frags == n_frags) break; rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); } skb->data_len = skb->len; skb->truesize += n_frags * efx->rx_buffer_truesize; skb_record_rx_queue(skb, channel->rx_queue.core_index); skb_mark_napi_id(skb, &channel->napi_str); gro_result = napi_gro_frags(napi); if (gro_result != GRO_DROP) channel->irq_mod_score += 2; }
static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb) { struct net_device_context *net_device_ctx = netdev_priv(ndev); struct hv_device *hdev = net_device_ctx->device_ctx; struct netvsc_device *nvsc_dev = hv_get_drvdata(hdev); u32 hash; u16 q_idx = 0; if (nvsc_dev == NULL || ndev->real_num_tx_queues <= 1) return 0; if (netvsc_set_hash(&hash, skb)) { q_idx = nvsc_dev->send_table[hash % VRSS_SEND_TAB_SIZE] % ndev->real_num_tx_queues; skb_set_hash(skb, hash, 0); } return q_idx; }
/** Routine to push packets arriving on Octeon interface upto network layer. * @param oct_id - octeon device id. * @param skbuff - skbuff struct to be passed to network layer. * @param len - size of total data received. * @param rh - Control header associated with the packet * @param param - additional control data with the packet * @param arg - farg registered in droq_ops */ static void liquidio_push_packet(u32 octeon_id __attribute__((unused)), void *skbuff, u32 len, union octeon_rh *rh, void *param, void *arg) { struct net_device *netdev = (struct net_device *)arg; struct octeon_droq *droq = container_of(param, struct octeon_droq, napi); struct sk_buff *skb = (struct sk_buff *)skbuff; struct skb_shared_hwtstamps *shhwtstamps; struct napi_struct *napi = param; u16 vtag = 0; u32 r_dh_off; u64 ns; if (netdev) { struct lio *lio = GET_LIO(netdev); struct octeon_device *oct = lio->oct_dev; /* Do not proceed if the interface is not in RUNNING state. */ if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) { recv_buffer_free(skb); droq->stats.rx_dropped++; return; } skb->dev = netdev; skb_record_rx_queue(skb, droq->q_no); if (likely(len > MIN_SKB_SIZE)) { struct octeon_skb_page_info *pg_info; unsigned char *va; pg_info = ((struct octeon_skb_page_info *)(skb->cb)); if (pg_info->page) { /* For Paged allocation use the frags */ va = page_address(pg_info->page) + pg_info->page_offset; memcpy(skb->data, va, MIN_SKB_SIZE); skb_put(skb, MIN_SKB_SIZE); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, pg_info->page, pg_info->page_offset + MIN_SKB_SIZE, len - MIN_SKB_SIZE, LIO_RXBUFFER_SZ); } } else { struct octeon_skb_page_info *pg_info = ((struct octeon_skb_page_info *)(skb->cb)); skb_copy_to_linear_data(skb, page_address(pg_info->page) + pg_info->page_offset, len); skb_put(skb, len); put_page(pg_info->page); } r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT; if (oct->ptp_enable) { if (rh->r_dh.has_hwtstamp) { /* timestamp is included from the hardware at * the beginning of the packet. */ if (ifstate_check (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) { /* Nanoseconds are in the first 64-bits * of the packet. */ memcpy(&ns, (skb->data + r_dh_off), sizeof(ns)); r_dh_off -= BYTES_PER_DHLEN_UNIT; shhwtstamps = skb_hwtstamps(skb); shhwtstamps->hwtstamp = ns_to_ktime(ns + lio->ptp_adjust); } } } if (rh->r_dh.has_hash) { __be32 *hash_be = (__be32 *)(skb->data + r_dh_off); u32 hash = be32_to_cpu(*hash_be); skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); r_dh_off -= BYTES_PER_DHLEN_UNIT; } skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT); skb->protocol = eth_type_trans(skb, skb->dev); if ((netdev->features & NETIF_F_RXCSUM) && (((rh->r_dh.encap_on) && (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) || (!(rh->r_dh.encap_on) && (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED)))) /* checksum has already been verified */ skb->ip_summed = CHECKSUM_UNNECESSARY; else skb->ip_summed = CHECKSUM_NONE; /* Setting Encapsulation field on basis of status received * from the firmware */ if (rh->r_dh.encap_on) { skb->encapsulation = 1; skb->csum_level = 1; droq->stats.rx_vxlan++; } /* inbound VLAN tag */ if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && rh->r_dh.vlan) { u16 priority = rh->r_dh.priority; u16 vid = rh->r_dh.vlan; vtag = (priority << VLAN_PRIO_SHIFT) | vid; __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag); } napi_gro_receive(napi, skb); droq->stats.rx_bytes_received += len - rh->r_dh.len * BYTES_PER_DHLEN_UNIT; droq->stats.rx_pkts_received++; } else { recv_buffer_free(skb); } }
static int xlgmac_rx_poll(struct xlgmac_channel *channel, int budget) { struct xlgmac_pdata *pdata = channel->pdata; struct xlgmac_ring *ring = channel->rx_ring; struct net_device *netdev = pdata->netdev; unsigned int len, dma_desc_len, max_len; unsigned int context_next, context; struct xlgmac_desc_data *desc_data; struct xlgmac_pkt_info *pkt_info; unsigned int incomplete, error; struct xlgmac_hw_ops *hw_ops; unsigned int received = 0; struct napi_struct *napi; struct sk_buff *skb; int packet_count = 0; hw_ops = &pdata->hw_ops; /* Nothing to do if there isn't a Rx ring for this channel */ if (!ring) return 0; incomplete = 0; context_next = 0; napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi; desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur); pkt_info = &ring->pkt_info; while (packet_count < budget) { /* First time in loop see if we need to restore state */ if (!received && desc_data->state_saved) { skb = desc_data->state.skb; error = desc_data->state.error; len = desc_data->state.len; } else { memset(pkt_info, 0, sizeof(*pkt_info)); skb = NULL; error = 0; len = 0; } read_again: desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur); if (xlgmac_rx_dirty_desc(ring) > XLGMAC_RX_DESC_MAX_DIRTY) xlgmac_rx_refresh(channel); if (hw_ops->dev_read(channel)) break; received++; ring->cur++; incomplete = XLGMAC_GET_REG_BITS( pkt_info->attributes, RX_PACKET_ATTRIBUTES_INCOMPLETE_POS, RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN); context_next = XLGMAC_GET_REG_BITS( pkt_info->attributes, RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS, RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN); context = XLGMAC_GET_REG_BITS( pkt_info->attributes, RX_PACKET_ATTRIBUTES_CONTEXT_POS, RX_PACKET_ATTRIBUTES_CONTEXT_LEN); /* Earlier error, just drain the remaining data */ if ((incomplete || context_next) && error) goto read_again; if (error || pkt_info->errors) { if (pkt_info->errors) netif_err(pdata, rx_err, netdev, "error in received packet\n"); dev_kfree_skb(skb); goto next_packet; } if (!context) { /* Length is cumulative, get this descriptor's length */ dma_desc_len = desc_data->rx.len - len; len += dma_desc_len; if (dma_desc_len && !skb) { skb = xlgmac_create_skb(pdata, napi, desc_data, dma_desc_len); if (!skb) error = 1; } else if (dma_desc_len) { dma_sync_single_range_for_cpu( pdata->dev, desc_data->rx.buf.dma_base, desc_data->rx.buf.dma_off, desc_data->rx.buf.dma_len, DMA_FROM_DEVICE); skb_add_rx_frag( skb, skb_shinfo(skb)->nr_frags, desc_data->rx.buf.pa.pages, desc_data->rx.buf.pa.pages_offset, dma_desc_len, desc_data->rx.buf.dma_len); desc_data->rx.buf.pa.pages = NULL; } } if (incomplete || context_next) goto read_again; if (!skb) goto next_packet; /* Be sure we don't exceed the configured MTU */ max_len = netdev->mtu + ETH_HLEN; if (!(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && (skb->protocol == htons(ETH_P_8021Q))) max_len += VLAN_HLEN; if (skb->len > max_len) { netif_err(pdata, rx_err, netdev, "packet length exceeds configured MTU\n"); dev_kfree_skb(skb); goto next_packet; } if (netif_msg_pktdata(pdata)) xlgmac_print_pkt(netdev, skb, false); skb_checksum_none_assert(skb); if (XLGMAC_GET_REG_BITS(pkt_info->attributes, RX_PACKET_ATTRIBUTES_CSUM_DONE_POS, RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN)) skb->ip_summed = CHECKSUM_UNNECESSARY; if (XLGMAC_GET_REG_BITS(pkt_info->attributes, RX_PACKET_ATTRIBUTES_VLAN_CTAG_POS, RX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN)) { __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), pkt_info->vlan_ctag); pdata->stats.rx_vlan_packets++; } if (XLGMAC_GET_REG_BITS(pkt_info->attributes, RX_PACKET_ATTRIBUTES_RSS_HASH_POS, RX_PACKET_ATTRIBUTES_RSS_HASH_LEN)) skb_set_hash(skb, pkt_info->rss_hash, pkt_info->rss_hash_type); skb->dev = netdev; skb->protocol = eth_type_trans(skb, netdev); skb_record_rx_queue(skb, channel->queue_index); napi_gro_receive(napi, skb); next_packet: packet_count++; } /* Check if we need to save state before leaving */ if (received && (incomplete || context_next)) { desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur); desc_data->state_saved = 1; desc_data->state.skb = skb; desc_data->state.len = len; desc_data->state.error = error; } XLGMAC_PR("packet_count = %d\n", packet_count); return packet_count; }